diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d633329ac9..5f17bb7e4c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,10 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 + - name: Set workaround for libaio on Ubuntu 24.04 (see https://askubuntu.com/questions/1512196/libaio1-on-noble/1512197#1512197) + run: | + sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1 + if: ${{ matrix.os == 'ubuntu-latest' }} - name: Set environment variable for Linux run: echo "RunAzureTests=yes" >> $GITHUB_ENV if: ${{ matrix.os == 'ubuntu-latest' }} diff --git a/libs/common/RespWriteUtils.cs b/libs/common/RespWriteUtils.cs index 557fc09fba..db3b432eb9 100644 --- a/libs/common/RespWriteUtils.cs +++ b/libs/common/RespWriteUtils.cs @@ -8,6 +8,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; +using Tsavorite.core; namespace Garnet.common { @@ -697,6 +698,27 @@ public static bool WriteArrayWithNullElements(int len, ref byte* curr, byte* end return true; } + /// + /// Writes an array consisting of an ETag followed by a Bulk string value into the buffer. + /// NOTE: Caller should make sure there is enough space in the buffer for sending the etag, and value array. + /// + /// + /// + /// + /// + /// + public static void WriteEtagValArray(long etag, ref ReadOnlySpan value, ref byte* curr, byte* end, bool writeDirect) + { + // Writes a Resp encoded Array of Integer for ETAG as first element, and bulk string for value as second element + RespWriteUtils.WriteArrayLength(2, ref curr, end); + RespWriteUtils.WriteInteger(etag, ref curr, end); + + if (writeDirect) + RespWriteUtils.WriteDirect(value, ref curr, end); + else + RespWriteUtils.WriteBulkString(value, ref curr, end); + } + /// /// Write newline (\r\n) to /// diff --git a/libs/resources/RespCommandsDocs.json b/libs/resources/RespCommandsDocs.json index 1b77ab83bb..f2cde1e0b8 100644 --- a/libs/resources/RespCommandsDocs.json +++ b/libs/resources/RespCommandsDocs.json @@ -4498,6 +4498,45 @@ "DisplayText": "newkey", "Type": "Key", "KeySpecIndex": 1 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "WITHETAG", + "DisplayText": "WITHETAG", + "Type": "PureToken", + "ArgumentFlags": "Optional", + "Token": "WITHETAG" + } + ] + }, + { + "Command": "RENAMENX", + "Name": "RENAMENX", + "Summary": "Renames a key and overwrites the destination if the newkey does not exist.", + "Group": "Generic", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "NEWKEY", + "DisplayText": "newkey", + "Type": "Key", + "KeySpecIndex": 1 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "WITHETAG", + "DisplayText": "WITHETAG", + "Type": "PureToken", + "ArgumentFlags": "Optional", + "Token": "WITHETAG" } ] }, @@ -4928,6 +4967,14 @@ "Token": "GET", "ArgumentFlags": "Optional" }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "WITHETAG", + "DisplayText": "WITHETAG", + "Type": "PureToken", + "ArgumentFlags": "Optional", + "Token": "WITHETAG" + }, { "TypeDiscriminator": "RespCommandContainerArgument", "Name": "EXPIRATION", @@ -4973,6 +5020,94 @@ } ] }, + { + "Command": "GETIFNOTMATCH", + "Name": "GETIFNOTMATCH", + "Summary": "Gets the ETag and value if the key\u0027s current etag does not match the given etag.", + "Group": "String", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "ETAG", + "DisplayText": "etag", + "Type": "Integer" + } + ] + }, + { + "Command": "GETWITHETAG", + "Name": "GETWITHETAG", + "Summary": "Gets the ETag and value for the key", + "Group": "String", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + } + ] + }, + { + "Command": "SETIFMATCH", + "Name": "SETIFMATCH", + "Summary": "Sets the string value of a key, ignoring its type, if the key\u0027s current etag matches the given etag.", + "Group": "String", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "VALUE", + "DisplayText": "value", + "Type": "String" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "ETAG", + "DisplayText": "etag", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "EXPIRATION", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "SECONDS", + "DisplayText": "seconds", + "Type": "Integer", + "Token": "EX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MILLISECONDS", + "DisplayText": "milliseconds", + "Type": "Integer", + "Token": "PX" + } + ] + } + ] + }, { "Command": "SETBIT", "Name": "SETBIT", diff --git a/libs/resources/RespCommandsInfo.json b/libs/resources/RespCommandsInfo.json index 52bd1123c7..184538ffb0 100644 --- a/libs/resources/RespCommandsInfo.json +++ b/libs/resources/RespCommandsInfo.json @@ -1496,6 +1496,32 @@ } ] }, + { + "Command": "GETIFNOTMATCH", + "Name": "GETIFNOTMATCH", + "IsInternal": false, + "Arity": 3, + "Flags": "NONE", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, String, Read", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ] + }, { "Command": "GETRANGE", "Name": "GETRANGE", @@ -1546,6 +1572,32 @@ } ] }, + { + "Command": "GETWITHETAG", + "Name": "GETWITHETAG", + "IsInternal": false, + "Arity": 2, + "Flags": "NONE", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, String, Read", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ] + }, { "Command": "HDEL", "Name": "HDEL", @@ -3006,7 +3058,7 @@ { "Command": "RENAME", "Name": "RENAME", - "Arity": 3, + "Arity": -3, "Flags": "Write", "FirstKey": 1, "LastKey": 2, @@ -3044,7 +3096,7 @@ { "Command": "RENAMENX", "Name": "RENAMENX", - "Arity": 3, + "Arity": -3, "Flags": "Fast, Write", "FirstKey": 1, "LastKey": 2, @@ -3500,6 +3552,32 @@ } ] }, + { + "Command": "SETIFMATCH", + "Name": "SETIFMATCH", + "IsInternal": false, + "Arity": -4, + "Flags": "NONE", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Slow, String, Write", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Access, Update, VariableFlags" + } + ] + }, { "Command": "SETRANGE", "Name": "SETRANGE", diff --git a/libs/server/API/GarnetApi.cs b/libs/server/API/GarnetApi.cs index 9a2d797212..23246de16c 100644 --- a/libs/server/API/GarnetApi.cs +++ b/libs/server/API/GarnetApi.cs @@ -180,12 +180,12 @@ public GarnetStatus APPEND(ArgSlice key, ArgSlice value, ref ArgSlice output) #region RENAME /// - public GarnetStatus RENAME(ArgSlice oldKey, ArgSlice newKey, StoreType storeType = StoreType.All) - => storageSession.RENAME(oldKey, newKey, storeType); + public GarnetStatus RENAME(ArgSlice oldKey, ArgSlice newKey, bool withEtag = false, StoreType storeType = StoreType.All) + => storageSession.RENAME(oldKey, newKey, storeType, withEtag); /// - public GarnetStatus RENAMENX(ArgSlice oldKey, ArgSlice newKey, out int result, StoreType storeType = StoreType.All) - => storageSession.RENAMENX(oldKey, newKey, storeType, out result); + public GarnetStatus RENAMENX(ArgSlice oldKey, ArgSlice newKey, out int result, bool withEtag = false, StoreType storeType = StoreType.All) + => storageSession.RENAMENX(oldKey, newKey, storeType, out result, withEtag); #endregion #region EXISTS diff --git a/libs/server/API/IGarnetApi.cs b/libs/server/API/IGarnetApi.cs index 15aad7a4e7..ca7198aa09 100644 --- a/libs/server/API/IGarnetApi.cs +++ b/libs/server/API/IGarnetApi.cs @@ -131,7 +131,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus RENAME(ArgSlice oldKey, ArgSlice newKey, StoreType storeType = StoreType.All); + GarnetStatus RENAME(ArgSlice oldKey, ArgSlice newKey, bool withEtag, StoreType storeType = StoreType.All); /// /// Renames key to newkey if newkey does not yet exist. It returns an error when key does not exist. @@ -141,7 +141,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// The result of the operation. /// The type of store to perform the operation on. /// - GarnetStatus RENAMENX(ArgSlice oldKey, ArgSlice newKey, out int result, StoreType storeType = StoreType.All); + GarnetStatus RENAMENX(ArgSlice oldKey, ArgSlice newKey, out int result, bool withEtag, StoreType storeType = StoreType.All); #endregion #region EXISTS diff --git a/libs/server/Constants.cs b/libs/server/Constants.cs new file mode 100644 index 0000000000..7337b6a714 --- /dev/null +++ b/libs/server/Constants.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + + +namespace Garnet.server +{ + internal static class Constants + { + public const byte EtagSize = sizeof(long); + + public const long BaseEtag = 0; + } +} \ No newline at end of file diff --git a/libs/server/Custom/CustomFunctions.cs b/libs/server/Custom/CustomFunctions.cs index b9ce4f8a01..df5976baf1 100644 --- a/libs/server/Custom/CustomFunctions.cs +++ b/libs/server/Custom/CustomFunctions.cs @@ -116,6 +116,17 @@ protected static unsafe void WriteBulkStringArray(ref MemoryResult output, } } + /// + /// Create output as bulk string, from given Span + /// + protected static unsafe void WriteBulkString(ref MemoryResult output, Span simpleString) + { + var _output = (output.MemoryOwner, output.Length); + WriteBulkString(ref _output, simpleString); + output.MemoryOwner = _output.MemoryOwner; + output.Length = _output.Length; + } + /// /// Create output as bulk string, from given Span /// diff --git a/libs/server/InputHeader.cs b/libs/server/InputHeader.cs index 3a9ed6c7ee..5abff66a89 100644 --- a/libs/server/InputHeader.cs +++ b/libs/server/InputHeader.cs @@ -11,10 +11,18 @@ namespace Garnet.server { /// /// Flags used by append-only file (AOF/WAL) + /// The byte representation only use the last 3 bits of the byte since the lower 5 bits of the field used to store the flag stores other data in the case of Object types. + /// In the case of a Rawstring, the last 4 bits are used for flags, and the other 4 bits are unused of the byte. + /// NOTE: This will soon be expanded as a part of a breaking change to make WithEtag bit compatible with object store as well. /// [Flags] public enum RespInputFlags : byte { + /// + /// Flag indicating an operation intending to add an etag for a RAWSTRING command. + /// + WithEtag = 16, + /// /// Flag indicating a SET operation that returns the previous value /// @@ -39,6 +47,9 @@ public struct RespInputHeader /// Size of header /// public const int Size = 3; + + // Since we know WithEtag is not used with any Object types, we keep the flag mask to work with the last 3 bits as flags, + // and the other 5 bits for storing object associated flags. However, in the case of Rawstring we use the last 4 bits for flags, and let the others remain unused. internal const byte FlagMask = (byte)RespInputFlags.SetGet - 1; [FieldOffset(0)] @@ -123,6 +134,22 @@ internal ListOperation ListOp /// internal unsafe void SetSetGetFlag() => flags |= RespInputFlags.SetGet; + /// + /// Set "WithEtag" flag for the input header + /// + internal void SetWithEtagFlag() => flags |= RespInputFlags.WithEtag; + + /// + /// Check if the WithEtag flag is set + /// + /// + internal bool CheckWithEtagFlag() => (flags & RespInputFlags.WithEtag) != 0; + + /// + /// Check that neither SetGet nor WithEtag flag is set + /// + internal bool NotSetGetNorCheckWithEtag() => (flags & (RespInputFlags.SetGet | RespInputFlags.WithEtag)) == 0; + /// /// Check if record is expired, either deterministically during log replay, /// or based on current time in normal operation. diff --git a/libs/server/Resp/BasicCommands.cs b/libs/server/Resp/BasicCommands.cs index 860dd691b2..e86bdf8489 100644 --- a/libs/server/Resp/BasicCommands.cs +++ b/libs/server/Resp/BasicCommands.cs @@ -3,7 +3,6 @@ using System; using System.Diagnostics; -using System.Runtime.CompilerServices; using System.Text; using System.Threading.Tasks; using Garnet.common; @@ -440,6 +439,12 @@ private bool NetworkSETNX(bool highPrecision, ref TGarnetApi storage return NetworkSETEXNX(ref storageApi); } + enum EtagOption : byte + { + None, + WithETag, + } + enum ExpirationOption : byte { None, @@ -458,7 +463,7 @@ enum ExistOptions : byte } /// - /// SET EX NX + /// SET EX NX [WITHETAG] /// private bool NetworkSETEXNX(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi @@ -473,6 +478,7 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) ReadOnlySpan errorMessage = default; var existOptions = ExistOptions.None; var expOption = ExpirationOption.None; + var etagOption = EtagOption.None; var getValue = false; var tokenIdx = 2; @@ -561,9 +567,32 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) } else if (nextOpt.SequenceEqual(CmdStrings.GET)) { - tokenIdx++; + if (etagOption != EtagOption.None) + { + // cannot do withEtag and getValue since withEtag SET already returns ETag in response + errorMessage = CmdStrings.RESP_ERR_WITHETAG_AND_GETVALUE; + break; + } + getValue = true; } + else if (nextOpt.SequenceEqual(CmdStrings.WITHETAG)) + { + if (etagOption != EtagOption.None) + { + errorMessage = CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR; + break; + } + + if (getValue) + { + // cannot do withEtag and getValue since withEtag SET already returns ETag in response + errorMessage = CmdStrings.RESP_ERR_WITHETAG_AND_GETVALUE; + break; + } + + etagOption = EtagOption.WithETag; + } else { if (!optUpperCased) @@ -587,60 +616,45 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) return true; } + bool withEtag = etagOption == EtagOption.WithETag; + + bool isHighPrecision = expOption == ExpirationOption.PX; + switch (expOption) { case ExpirationOption.None: case ExpirationOption.EX: - switch (existOptions) - { - case ExistOptions.None: - return getValue - ? NetworkSET_Conditional(RespCommand.SET, expiry, ref sbKey, true, - false, ref storageApi) - : NetworkSET_EX(RespCommand.SET, expOption, expiry, ref sbKey, ref sbVal, ref storageApi); // Can perform a blind update - case ExistOptions.XX: - return NetworkSET_Conditional(RespCommand.SETEXXX, expiry, ref sbKey, getValue, false, - ref storageApi); - case ExistOptions.NX: - return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, ref sbKey, getValue, false, - ref storageApi); - } - - break; case ExpirationOption.PX: switch (existOptions) { case ExistOptions.None: - return getValue - ? NetworkSET_Conditional(RespCommand.SET, expiry, ref sbKey, true, - true, ref storageApi) + return getValue || withEtag + ? NetworkSET_Conditional(RespCommand.SET, expiry, ref sbKey, getValue, + isHighPrecision, withEtag, ref storageApi) : NetworkSET_EX(RespCommand.SET, expOption, expiry, ref sbKey, ref sbVal, ref storageApi); // Can perform a blind update case ExistOptions.XX: - return NetworkSET_Conditional(RespCommand.SETEXXX, expiry, ref sbKey, getValue, true, - ref storageApi); + return NetworkSET_Conditional(RespCommand.SETEXXX, expiry, ref sbKey, + getValue, isHighPrecision, withEtag, ref storageApi); case ExistOptions.NX: - return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, ref sbKey, getValue, true, - ref storageApi); + return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, ref sbKey, + getValue, isHighPrecision, withEtag, ref storageApi); } - break; - case ExpirationOption.KEEPTTL: Debug.Assert(expiry == 0); // no expiration if KEEPTTL switch (existOptions) { case ExistOptions.None: // We can never perform a blind update due to KEEPTTL - return NetworkSET_Conditional(RespCommand.SETKEEPTTL, expiry, ref sbKey, getValue, false, - ref storageApi); + return NetworkSET_Conditional(RespCommand.SETKEEPTTL, expiry, ref sbKey + , getValue, highPrecision: false, withEtag, ref storageApi); case ExistOptions.XX: - return NetworkSET_Conditional(RespCommand.SETKEEPTTLXX, expiry, ref sbKey, getValue, false, - ref storageApi); + return NetworkSET_Conditional(RespCommand.SETKEEPTTLXX, expiry, ref sbKey, + getValue, highPrecision: false, withEtag, ref storageApi); case ExistOptions.NX: - return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, ref sbKey, getValue, false, - ref storageApi); + return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, ref sbKey, + getValue, highPrecision: false, withEtag, ref storageApi); } - break; } @@ -670,7 +684,7 @@ private unsafe bool NetworkSET_EX(RespCommand cmd, ExpirationOption return true; } - private bool NetworkSET_Conditional(RespCommand cmd, int expiry, ref SpanByte key, bool getValue, bool highPrecision, ref TGarnetApi storageApi) + private bool NetworkSET_Conditional(RespCommand cmd, int expiry, ref SpanByte key, bool getValue, bool highPrecision, bool withEtag, ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { var inputArg = expiry == 0 @@ -682,54 +696,61 @@ private bool NetworkSET_Conditional(RespCommand cmd, int expiry, ref var input = new RawStringInput(cmd, ref parseState, startIdx: 1, arg1: inputArg); - if (getValue) - input.header.SetSetGetFlag(); - - if (getValue) + if (!getValue && !withEtag) { - var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); - var status = storageApi.SET_Conditional(ref key, - ref input, ref o); + // the following debug assertion is the catch any edge case leading to SETIFMATCH skipping the above block + Debug.Assert(cmd != RespCommand.SETIFMATCH, "SETIFMATCH should have gone though pointing to right output variable"); + + GarnetStatus status = storageApi.SET_Conditional(ref key, ref input); + + bool ok = status != GarnetStatus.NOTFOUND; + + if (cmd == RespCommand.SETEXNX) + ok = !ok; - // Status tells us whether an old image was found during RMW or not - if (status == GarnetStatus.NOTFOUND) + if (ok) { - Debug.Assert(o.IsSpanByte); - while (!RespWriteUtils.WriteNull(ref dcurr, dend)) + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) SendAndReset(); } else { - if (!o.IsSpanByte) - SendAndReset(o.Memory, o.Length); - else - dcurr += o.Length; + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_ERRNOTFOUND, ref dcurr, dend)) + SendAndReset(); } + return true; } else { - var status = storageApi.SET_Conditional(ref key, ref input); + if (withEtag) + input.header.SetWithEtagFlag(); - var ok = status != GarnetStatus.NOTFOUND; + if (getValue) + input.header.SetSetGetFlag(); - // Status tells us whether an old image was found during RMW or not - // For a "set if not exists", NOTFOUND means the operation succeeded - // So we invert the ok flag - if (cmd == RespCommand.SETEXNX) - ok = !ok; - if (!ok) + // anything with getValue or withEtag always writes to the buffer in the happy path + SpanByteAndMemory outputBuffer = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + GarnetStatus status = storageApi.SET_Conditional(ref key, + ref input, ref outputBuffer); + + // The data will be on the buffer either when we know the response is ok or when the withEtag flag is set. + bool ok = status != GarnetStatus.NOTFOUND || withEtag; + + if (ok) { - while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_ERRNOTFOUND, ref dcurr, dend)) - SendAndReset(); + if (!outputBuffer.IsSpanByte) + SendAndReset(outputBuffer.Memory, outputBuffer.Length); + else + dcurr += outputBuffer.Length; } else { - while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_ERRNOTFOUND, ref dcurr, dend)) SendAndReset(); } - } - return true; + return true; + } } /// diff --git a/libs/server/Resp/BasicEtagCommands.cs b/libs/server/Resp/BasicEtagCommands.cs new file mode 100644 index 0000000000..607fd27d46 --- /dev/null +++ b/libs/server/Resp/BasicEtagCommands.cs @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Text; +using System.Threading.Tasks; +using Garnet.common; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Server session for RESP protocol - ETag associated commands are in this file + /// + internal sealed unsafe partial class RespServerSession : ServerSessionBase + { + /// + /// GETWITHETAG key + /// Given a key get the value and it's ETag + /// + private bool NetworkGETWITHETAG(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + Debug.Assert(parseState.Count == 1); + + var key = parseState.GetArgSliceByRef(0).SpanByte; + var input = new RawStringInput(RespCommand.GETWITHETAG, ref parseState, startIdx: 1); + var output = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + var status = storageApi.GET(ref key, ref input, ref output); + + switch (status) + { + case GarnetStatus.NOTFOUND: + Debug.Assert(output.IsSpanByte); + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_ERRNOTFOUND, ref dcurr, dend)) + SendAndReset(); + break; + default: + if (!output.IsSpanByte) + SendAndReset(output.Memory, output.Length); + else + dcurr += output.Length; + break; + } + + return true; + } + + /// + /// GETIFNOTMATCH key etag + /// Given a key and an etag, return the value and it's etag. + /// + private bool NetworkGETIFNOTMATCH(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + Debug.Assert(parseState.Count == 2); + + var key = parseState.GetArgSliceByRef(0).SpanByte; + var input = new RawStringInput(RespCommand.GETIFNOTMATCH, ref parseState, startIdx: 1); + var output = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + var status = storageApi.GET(ref key, ref input, ref output); + + switch (status) + { + case GarnetStatus.NOTFOUND: + Debug.Assert(output.IsSpanByte); + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_ERRNOTFOUND, ref dcurr, dend)) + SendAndReset(); + break; + default: + if (!output.IsSpanByte) + SendAndReset(output.Memory, output.Length); + else + dcurr += output.Length; + break; + } + + return true; + } + + /// + /// SETIFMATCH key val etag EX|PX expiry + /// Sets a key value pair only if the already existing etag matches the etag sent as a part of the request. + /// + /// + /// + /// + private bool NetworkSETIFMATCH(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (parseState.Count < 3 || parseState.Count > 5) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.SETIFMATCH)); + } + + // SETIFMATCH Args: KEY VAL ETAG -> [ ((EX || PX) expiration)] + int expiry = 0; + ReadOnlySpan errorMessage = default; + var expOption = ExpirationOption.None; + + var tokenIdx = 3; + Span nextOpt = default; + var optUpperCased = false; + while (tokenIdx < parseState.Count || optUpperCased) + { + if (!optUpperCased) + { + nextOpt = parseState.GetArgSliceByRef(tokenIdx++).Span; + } + + if (nextOpt.SequenceEqual(CmdStrings.EX)) + { + // Validate expiry + if (!parseState.TryGetInt(tokenIdx++, out expiry)) + { + errorMessage = CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER; + break; + } + + if (expOption != ExpirationOption.None) + { + errorMessage = CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR; + break; + } + + expOption = ExpirationOption.EX; + if (expiry <= 0) + { + errorMessage = CmdStrings.RESP_ERR_GENERIC_INVALIDEXP_IN_SET; + break; + } + } + else if (nextOpt.SequenceEqual(CmdStrings.PX)) + { + // Validate expiry + if (!parseState.TryGetInt(tokenIdx++, out expiry)) + { + errorMessage = CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER; + break; + } + + if (expOption != ExpirationOption.None) + { + errorMessage = CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR; + break; + } + + expOption = ExpirationOption.PX; + if (expiry <= 0) + { + errorMessage = CmdStrings.RESP_ERR_GENERIC_INVALIDEXP_IN_SET; + break; + } + } + else + { + if (!optUpperCased) + { + AsciiUtils.ToUpperInPlace(nextOpt); + optUpperCased = true; + continue; + } + + errorMessage = CmdStrings.RESP_ERR_GENERIC_UNK_CMD; + break; + } + + optUpperCased = false; + } + + if (!errorMessage.IsEmpty) + { + while (!RespWriteUtils.WriteError(errorMessage, ref dcurr, dend)) + SendAndReset(); + return true; + } + + SpanByte key = parseState.GetArgSliceByRef(0).SpanByte; + + NetworkSET_Conditional(RespCommand.SETIFMATCH, expiry, ref key, getValue: true, highPrecision: expOption == ExpirationOption.PX, withEtag: true, ref storageApi); + + return true; + } + } +} \ No newline at end of file diff --git a/libs/server/Resp/Bitmap/BitmapManagerBitPos.cs b/libs/server/Resp/Bitmap/BitmapManagerBitPos.cs index 2273a27c0f..aba15d649a 100644 --- a/libs/server/Resp/Bitmap/BitmapManagerBitPos.cs +++ b/libs/server/Resp/Bitmap/BitmapManagerBitPos.cs @@ -94,7 +94,9 @@ public static long BitPosDriver(byte setVal, long startOffset, long endOffset, b return -1; endOffset = endOffset >= valLen ? valLen : endOffset; - return BitPosByte(value, setVal, startOffset, endOffset); + long pos = BitPosByte(value, setVal, startOffset, endOffset); + // check if position is exceeding the last byte in acceptable range + return pos >= ((endOffset + 1) * 8) ? -1 : pos; } startOffset = startOffset < 0 ? ProcessNegativeOffset(startOffset, valLen * 8) : startOffset; @@ -119,7 +121,8 @@ public static long BitPosDriver(byte setVal, long startOffset, long endOffset, b var _startOffset = (startOffset / 8) + 1; var _endOffset = (endOffset / 8) - 1; var _bpos = BitPosByte(value, setVal, _startOffset, _endOffset); - if (_bpos != -1) return _bpos; + + if (_bpos != -1 && _bpos < (_endOffset + 1) * 8) return _bpos; // Search suffix var _spos = BitPosIndexBitSearch(value, setVal, endOffset); @@ -130,7 +133,7 @@ public static long BitPosDriver(byte setVal, long startOffset, long endOffset, b /// Find pos of set/clear bit in a sequence of bytes. /// /// Pointer to start of bitmap. - /// + /// The bit value to search for (0 for cleared bit or 1 for set bit). /// Starting offset into bitmap. /// End offset into bitmap. /// diff --git a/libs/server/Resp/CmdStrings.cs b/libs/server/Resp/CmdStrings.cs index 715ebc2f29..26b0b88a44 100644 --- a/libs/server/Resp/CmdStrings.cs +++ b/libs/server/Resp/CmdStrings.cs @@ -98,6 +98,7 @@ static partial class CmdStrings public static ReadOnlySpan KEEPTTL => "KEEPTTL"u8; public static ReadOnlySpan NX => "NX"u8; public static ReadOnlySpan XX => "XX"u8; + public static ReadOnlySpan WITHETAG => "WITHETAG"u8; public static ReadOnlySpan UNSAFETRUNCATELOG => "UNSAFETRUNCATELOG"u8; public static ReadOnlySpan SAMPLES => "SAMPLES"u8; public static ReadOnlySpan RANK => "RANK"u8; @@ -154,6 +155,7 @@ static partial class CmdStrings public static ReadOnlySpan RESP_ERR_WRONG_TYPE => "WRONGTYPE Operation against a key holding the wrong kind of value."u8; public static ReadOnlySpan RESP_ERR_WRONG_TYPE_HLL => "WRONGTYPE Key is not a valid HyperLogLog string value."u8; public static ReadOnlySpan RESP_ERR_EXEC_ABORT => "EXECABORT Transaction discarded because of previous errors."u8; + public static ReadOnlySpan RESP_ERR_ETAG_ON_CUSTOM_PROC => "WRONGTYPE Key with etag cannot be used for custom procedure."u8; /// /// Generic error response strings, i.e. these are of the form "-ERR error message\r\n" @@ -170,6 +172,7 @@ static partial class CmdStrings public static ReadOnlySpan RESP_ERR_GENERIC_WATCH_IN_MULTI => "ERR WATCH inside MULTI is not allowed"u8; public static ReadOnlySpan RESP_ERR_GENERIC_INVALIDEXP_IN_SET => "ERR invalid expire time in 'set' command"u8; public static ReadOnlySpan RESP_ERR_GENERIC_SYNTAX_ERROR => "ERR syntax error"u8; + public static ReadOnlySpan RESP_ERR_WITHETAG_AND_GETVALUE => "ERR WITHETAG option not allowed with GET inside of SET"u8; public static ReadOnlySpan RESP_ERR_GENERIC_OFFSETOUTOFRANGE => "ERR offset is out of range"u8; public static ReadOnlySpan RESP_ERR_GENERIC_BIT_IS_NOT_INTEGER => "ERR bit is not an integer or out of range"u8; public static ReadOnlySpan RESP_ERR_GENERIC_BITOFFSET_IS_NOT_INTEGER => "ERR bit offset is not an integer or out of range"u8; diff --git a/libs/server/Resp/KeyAdminCommands.cs b/libs/server/Resp/KeyAdminCommands.cs index 8cf1434517..1ce25dd487 100644 --- a/libs/server/Resp/KeyAdminCommands.cs +++ b/libs/server/Resp/KeyAdminCommands.cs @@ -17,14 +17,29 @@ internal sealed unsafe partial class RespServerSession : ServerSessionBase private bool NetworkRENAME(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { - if (parseState.Count != 2) + // one optional command for with etag + if (parseState.Count < 2 || parseState.Count > 3) { return AbortWithWrongNumberOfArguments(nameof(RespCommand.RENAME)); } var oldKeySlice = parseState.GetArgSliceByRef(0); var newKeySlice = parseState.GetArgSliceByRef(1); - var status = storageApi.RENAME(oldKeySlice, newKeySlice); + + var withEtag = false; + if (parseState.Count == 3) + { + if (!parseState.GetArgSliceByRef(2).ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.WITHETAG)) + { + while (!RespWriteUtils.WriteError($"ERR Unsupported option {parseState.GetString(2)}", ref dcurr, dend)) + SendAndReset(); + return true; + } + + withEtag = true; + } + + var status = storageApi.RENAME(oldKeySlice, newKeySlice, withEtag); switch (status) { @@ -46,14 +61,29 @@ private bool NetworkRENAME(ref TGarnetApi storageApi) private bool NetworkRENAMENX(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { - if (parseState.Count != 2) + // one optional command for with etag + if (parseState.Count < 2 || parseState.Count > 3) { return AbortWithWrongNumberOfArguments(nameof(RespCommand.RENAMENX)); } var oldKeySlice = parseState.GetArgSliceByRef(0); var newKeySlice = parseState.GetArgSliceByRef(1); - var status = storageApi.RENAMENX(oldKeySlice, newKeySlice, out var result); + + var withEtag = false; + if (parseState.Count == 3) + { + if (!parseState.GetArgSliceByRef(2).ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.WITHETAG)) + { + while (!RespWriteUtils.WriteError($"ERR Unsupported option {parseState.GetString(2)}", ref dcurr, dend)) + SendAndReset(); + return true; + } + + withEtag = true; + } + + var status = storageApi.RENAMENX(oldKeySlice, newKeySlice, out var result, withEtag); if (status == GarnetStatus.OK) { diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index 8d288da07f..2e4c53d7a4 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -33,7 +33,9 @@ public enum RespCommand : ushort GEOSEARCH, GET, GETBIT, + GETIFNOTMATCH, GETRANGE, + GETWITHETAG, HEXISTS, HGET, HGETALL, @@ -153,6 +155,7 @@ public enum RespCommand : ushort SETEXNX, SETEXXX, SETNX, + SETIFMATCH, SETKEEPTTL, SETKEEPTTLXX, SETRANGE, @@ -432,7 +435,7 @@ public static bool IsAofIndependent(this RespCommand cmd) /// /// Turns any not-quite-a-real-command entries in into the equivalent command - /// for ACL'ing purposes. + /// for ACL'ing purposes and reading command info purposes /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static RespCommand NormalizeForACLs(this RespCommand cmd) @@ -676,8 +679,6 @@ private RespCommand FastParseCommand(out int count) (2 << 4) | 5 when lastWord == MemoryMarshal.Read("\nPFADD\r\n"u8) => RespCommand.PFADD, (2 << 4) | 6 when lastWord == MemoryMarshal.Read("INCRBY\r\n"u8) => RespCommand.INCRBY, (2 << 4) | 6 when lastWord == MemoryMarshal.Read("DECRBY\r\n"u8) => RespCommand.DECRBY, - (2 << 4) | 6 when lastWord == MemoryMarshal.Read("RENAME\r\n"u8) => RespCommand.RENAME, - (2 << 4) | 8 when lastWord == MemoryMarshal.Read("NAMENX\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("RE"u8) => RespCommand.RENAMENX, (2 << 4) | 6 when lastWord == MemoryMarshal.Read("GETBIT\r\n"u8) => RespCommand.GETBIT, (2 << 4) | 6 when lastWord == MemoryMarshal.Read("APPEND\r\n"u8) => RespCommand.APPEND, (2 << 4) | 6 when lastWord == MemoryMarshal.Read("GETSET\r\n"u8) => RespCommand.GETSET, @@ -694,7 +695,9 @@ private RespCommand FastParseCommand(out int count) _ => ((length << 4) | count) switch { // Commands with dynamic number of arguments - >= ((3 << 4) | 3) and <= ((3 << 4) | 6) when lastWord == MemoryMarshal.Read("3\r\nSET\r\n"u8) => RespCommand.SETEXNX, + >= ((6 << 4) | 2) and <= ((6 << 4) | 3) when lastWord == MemoryMarshal.Read("RENAME\r\n"u8) => RespCommand.RENAME, + >= ((8 << 4) | 2) and <= ((8 << 4) | 3) when lastWord == MemoryMarshal.Read("NAMENX\r\n"u8) && *(ushort*)(ptr + 8) == MemoryMarshal.Read("RE"u8) => RespCommand.RENAMENX, + >= ((3 << 4) | 3) and <= ((3 << 4) | 7) when lastWord == MemoryMarshal.Read("3\r\nSET\r\n"u8) => RespCommand.SETEXNX, >= ((5 << 4) | 1) and <= ((5 << 4) | 3) when lastWord == MemoryMarshal.Read("\nGETEX\r\n"u8) => RespCommand.GETEX, >= ((6 << 4) | 0) and <= ((6 << 4) | 9) when lastWord == MemoryMarshal.Read("RUNTXP\r\n"u8) => RespCommand.RUNTXP, >= ((6 << 4) | 2) and <= ((6 << 4) | 3) when lastWord == MemoryMarshal.Read("EXPIRE\r\n"u8) => RespCommand.EXPIRE, @@ -1427,6 +1430,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.SDIFFSTORE; } + else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nSETI"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("FMATCH\r\n"u8)) + { + return RespCommand.SETIFMATCH; + } else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nEXPI"u8) && *(ulong*)(ptr + 9) == MemoryMarshal.Read("RETIME\r\n"u8)) { return RespCommand.EXPIRETIME; @@ -1452,7 +1459,6 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan return RespCommand.ZINTERCARD; } break; - case 11: if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nUNSUB"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("SCRIBE\r\n"u8)) { @@ -1478,6 +1484,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.SINTERSTORE; } + else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nGETWI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("THETAG\r\n"u8)) + { + return RespCommand.GETWITHETAG; + } else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nPEXPI"u8) && *(ulong*)(ptr + 10) == MemoryMarshal.Read("RETIME\r\n"u8)) { return RespCommand.PEXPIRETIME; @@ -1516,6 +1526,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.ZRANGEBYSCORE; } + else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("\nGETIFNO"u8) && *(ulong*)(ptr + 12) == MemoryMarshal.Read("TMATCH\r\n"u8)) + { + return RespCommand.GETIFNOTMATCH; + } break; case 14: diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index eb9a144006..96588b341a 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -562,6 +562,10 @@ private bool ProcessBasicCommands(RespCommand cmd, ref TGarnetApi st RespCommand.READWRITE => NetworkREADWRITE(), RespCommand.EXPIREAT => NetworkEXPIREAT(RespCommand.EXPIREAT, ref storageApi), RespCommand.PEXPIREAT => NetworkEXPIREAT(RespCommand.PEXPIREAT, ref storageApi), + // Etag related commands + RespCommand.GETWITHETAG => NetworkGETWITHETAG(ref storageApi), + RespCommand.GETIFNOTMATCH => NetworkGETIFNOTMATCH(ref storageApi), + RespCommand.SETIFMATCH => NetworkSETIFMATCH(ref storageApi), _ => ProcessArrayCommands(cmd, ref storageApi) }; diff --git a/libs/server/Storage/Functions/EtagState.cs b/libs/server/Storage/Functions/EtagState.cs new file mode 100644 index 0000000000..c9f043f7fc --- /dev/null +++ b/libs/server/Storage/Functions/EtagState.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Indirection wrapper to provide a way to set offsets related to Etags and use the getters opaquely from outside. + /// + public struct EtagState + { + public EtagState() + { + } + + /// + /// Offset used accounting space for an etag during allocation + /// + public byte etagOffsetForVarlen { get; set; } = 0; + + /// + /// Gives an offset used to opaquely work with Etag in a payload. By calling this you can skip past the etag if it is present. + /// + public byte etagSkippedStart { get; private set; } = 0; + + /// + /// Resp response methods depend on the value for end being -1 or length of the payload. This field lets you work with providing the end opaquely. + /// + public int etagAccountedLength { get; private set; } = -1; + + /// + /// Field provides access to getting an Etag from a record, hiding whether it is actually present or not. + /// + public long etag { get; private set; } = Constants.BaseEtag; + + /// + /// Sets the values to indicate the presence of an Etag as a part of the payload value + /// + public static void SetValsForRecordWithEtag(ref EtagState curr, ref SpanByte value) + { + curr.etagOffsetForVarlen = Constants.EtagSize; + curr.etagSkippedStart = Constants.EtagSize; + curr.etagAccountedLength = value.LengthWithoutMetadata; + curr.etag = value.GetEtagInPayload(); + } + + public static void ResetState(ref EtagState curr) + { + curr.etagOffsetForVarlen = 0; + curr.etagSkippedStart = 0; + curr.etag = Constants.BaseEtag; + curr.etagAccountedLength = -1; + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/FunctionsState.cs b/libs/server/Storage/Functions/FunctionsState.cs index 055ad9f675..be097ca976 100644 --- a/libs/server/Storage/Functions/FunctionsState.cs +++ b/libs/server/Storage/Functions/FunctionsState.cs @@ -18,6 +18,7 @@ internal sealed class FunctionsState public readonly MemoryPool memoryPool; public readonly CacheSizeTracker objectStoreSizeTracker; public readonly GarnetObjectSerializer garnetObjectSerializer; + public EtagState etagState; public bool StoredProcMode; public FunctionsState(TsavoriteLog appendOnlyFile, WatchVersionMap watchVersionMap, CustomCommandManager customCommandManager, @@ -29,6 +30,7 @@ public FunctionsState(TsavoriteLog appendOnlyFile, WatchVersionMap watchVersionM this.memoryPool = memoryPool ?? MemoryPool.Shared; this.objectStoreSizeTracker = objectStoreSizeTracker; this.garnetObjectSerializer = garnetObjectSerializer; + this.etagState = new EtagState(); } public CustomRawStringFunctions GetCustomCommandFunctions(int id) diff --git a/libs/server/Storage/Functions/MainStore/DeleteMethods.cs b/libs/server/Storage/Functions/MainStore/DeleteMethods.cs index d94aa8b7eb..6c055bd368 100644 --- a/libs/server/Storage/Functions/MainStore/DeleteMethods.cs +++ b/libs/server/Storage/Functions/MainStore/DeleteMethods.cs @@ -13,6 +13,7 @@ namespace Garnet.server /// public bool SingleDeleter(ref SpanByte key, ref SpanByte value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) { + recordInfo.ClearHasETag(); functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash); return true; } @@ -27,6 +28,7 @@ public void PostSingleDeleter(ref SpanByte key, ref DeleteInfo deleteInfo) /// public bool ConcurrentDeleter(ref SpanByte key, ref SpanByte value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) { + recordInfo.ClearHasETag(); if (!deleteInfo.RecordInfo.Modified) functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash); if (functionsState.appendOnlyFile != null) diff --git a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs index 7b4ac193eb..747132d8af 100644 --- a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs +++ b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs @@ -92,7 +92,7 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB // This is accomplished by calling ConvertToHeap on the destination SpanByteAndMemory if (isFromPending) dst.ConvertToHeap(); - CopyRespTo(ref value, ref dst); + CopyRespTo(ref value, ref dst, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); break; case RespCommand.MIGRATE: @@ -122,20 +122,20 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB // Get value without RESP header; exclude expiration if (value.LengthWithoutMetadata <= dst.Length) { - dst.Length = value.LengthWithoutMetadata; - value.AsReadOnlySpan().CopyTo(dst.SpanByte.AsSpan()); + dst.Length = value.LengthWithoutMetadata - functionsState.etagState.etagSkippedStart; + value.AsReadOnlySpan(functionsState.etagState.etagSkippedStart).CopyTo(dst.SpanByte.AsSpan()); return; } dst.ConvertToHeap(); - dst.Length = value.LengthWithoutMetadata; + dst.Length = value.LengthWithoutMetadata - functionsState.etagState.etagSkippedStart; dst.Memory = functionsState.memoryPool.Rent(value.LengthWithoutMetadata); - value.AsReadOnlySpan().CopyTo(dst.Memory.Memory.Span); + value.AsReadOnlySpan(functionsState.etagState.etagSkippedStart).CopyTo(dst.Memory.Memory.Span); break; case RespCommand.GETBIT: var offset = input.parseState.GetLong(0); - var oldValSet = BitmapManager.GetBit(offset, value.ToPointer(), value.Length); + var oldValSet = BitmapManager.GetBit(offset, value.ToPointer() + functionsState.etagState.etagSkippedStart, value.Length - functionsState.etagState.etagSkippedStart); if (oldValSet == 0) CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref dst); else @@ -159,7 +159,7 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB } } - var count = BitmapManager.BitCountDriver(bcStartOffset, bcEndOffset, bcOffsetType, value.ToPointer(), value.Length); + var count = BitmapManager.BitCountDriver(bcStartOffset, bcEndOffset, bcOffsetType, value.ToPointer() + functionsState.etagState.etagSkippedStart, value.Length - functionsState.etagState.etagSkippedStart); CopyRespNumber(count, ref dst); break; @@ -185,13 +185,13 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB } var pos = BitmapManager.BitPosDriver(bpSetVal, bpStartOffset, bpEndOffset, bpOffsetType, - value.ToPointer(), value.Length); + value.ToPointer() + functionsState.etagState.etagSkippedStart, value.Length - functionsState.etagState.etagSkippedStart); *(long*)dst.SpanByte.ToPointer() = pos; CopyRespNumber(pos, ref dst); break; case RespCommand.BITOP: - var bitmap = (IntPtr)value.ToPointer(); + var bitmap = (IntPtr)value.ToPointer() + functionsState.etagState.etagSkippedStart; var output = dst.SpanByte.ToPointer(); *(long*)output = bitmap.ToInt64(); @@ -201,7 +201,7 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); - var (retValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, value.ToPointer(), value.Length); + var (retValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, value.ToPointer() + functionsState.etagState.etagSkippedStart, value.Length - functionsState.etagState.etagSkippedStart); if (!overflow) CopyRespNumber(retValue, ref dst); else @@ -236,14 +236,13 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB return; case RespCommand.GETRANGE: - var len = value.LengthWithoutMetadata; + var len = value.LengthWithoutMetadata - functionsState.etagState.etagSkippedStart; var start = input.parseState.GetInt(0); var end = input.parseState.GetInt(1); (start, end) = NormalizeRange(start, end, len); - CopyRespTo(ref value, ref dst, start, end); + CopyRespTo(ref value, ref dst, start + functionsState.etagState.etagSkippedStart, end + functionsState.etagState.etagSkippedStart); return; - case RespCommand.EXPIRETIME: var expireTime = ConvertUtils.UnixTimeInSecondsFromTicks(value.MetadataSize > 0 ? value.ExtraMetadata : -1); CopyRespNumber(expireTime, ref dst); @@ -402,48 +401,50 @@ void EvaluateExpireCopyUpdate(ExpireOption optionType, bool expiryExists, long n internal static bool CheckExpiry(ref SpanByte src) => src.ExtraMetadata < DateTimeOffset.UtcNow.Ticks; - static bool InPlaceUpdateNumber(long val, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + static bool InPlaceUpdateNumber(long val, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, int valueOffset) { var fNeg = false; var ndigits = NumUtils.NumDigitsInLong(val, ref fNeg); ndigits += fNeg ? 1 : 0; - if (ndigits > value.LengthWithoutMetadata) + if (ndigits > value.LengthWithoutMetadata - valueOffset) return false; rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.ShrinkSerializedLength(ndigits + value.MetadataSize); - _ = NumUtils.LongToSpanByte(val, value.AsSpan()); + value.ShrinkSerializedLength(ndigits + value.MetadataSize + valueOffset); + _ = NumUtils.LongToSpanByte(val, value.AsSpan(valueOffset)); rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); Debug.Assert(output.IsSpanByte, "This code assumes it is called in-place and did not go pending"); - value.AsReadOnlySpan().CopyTo(output.SpanByte.AsSpan()); - output.SpanByte.Length = value.LengthWithoutMetadata; + value.AsReadOnlySpan(valueOffset).CopyTo(output.SpanByte.AsSpan()); + output.SpanByte.Length = value.LengthWithoutMetadata - valueOffset; return true; } - static bool InPlaceUpdateNumber(double val, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + static bool InPlaceUpdateNumber(double val, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, int valueOffset) { var ndigits = NumUtils.NumOfCharInDouble(val, out var _, out var _, out var _); - if (ndigits > value.LengthWithoutMetadata) + if (ndigits > value.LengthWithoutMetadata - valueOffset) return false; rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.ShrinkSerializedLength(ndigits + value.MetadataSize); - _ = NumUtils.DoubleToSpanByte(val, value.AsSpan()); + value.ShrinkSerializedLength(ndigits + value.MetadataSize + valueOffset); + _ = NumUtils.DoubleToSpanByte(val, value.AsSpan(valueOffset)); rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); Debug.Assert(output.IsSpanByte, "This code assumes it is called in-place and did not go pending"); - value.AsReadOnlySpan().CopyTo(output.SpanByte.AsSpan()); - output.SpanByte.Length = value.LengthWithoutMetadata; + value.AsReadOnlySpan(valueOffset).CopyTo(output.SpanByte.AsSpan()); + output.SpanByte.Length = value.LengthWithoutMetadata - valueOffset; return true; } - static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, long input) + static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, long input, int valueOffset) { // Check if value contains a valid number - if (!IsValidNumber(value.LengthWithoutMetadata, value.ToPointer(), output.SpanByte.AsSpan(), out var val)) + int valLen = value.LengthWithoutMetadata - valueOffset; + byte* valPtr = value.ToPointer() + valueOffset; + if (!IsValidNumber(valLen, valPtr, output.SpanByte.AsSpan(), out var val)) return true; try @@ -456,13 +457,15 @@ static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory out return true; } - return InPlaceUpdateNumber(val, ref value, ref output, ref rmwInfo, ref recordInfo); + return InPlaceUpdateNumber(val, ref value, ref output, ref rmwInfo, ref recordInfo, valueOffset); } - static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, double input) + static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, double input, int valueOffset) { // Check if value contains a valid number - if (!IsValidDouble(value.LengthWithoutMetadata, value.ToPointer(), output.SpanByte.AsSpan(), out var val)) + int valLen = value.LengthWithoutMetadata - valueOffset; + byte* valPtr = value.ToPointer() + valueOffset; + if (!IsValidDouble(valLen, valPtr, output.SpanByte.AsSpan(), out var val)) return true; val += input; @@ -473,14 +476,21 @@ static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory out return true; } - return InPlaceUpdateNumber(val, ref value, ref output, ref rmwInfo, ref recordInfo); + return InPlaceUpdateNumber(val, ref value, ref output, ref rmwInfo, ref recordInfo, valueOffset); } - static void CopyUpdateNumber(long next, ref SpanByte newValue, ref SpanByteAndMemory output) + static void CopyUpdateNumber(long next, ref SpanByte newValue, ref SpanByteAndMemory output, int etagIgnoredOffset) { - NumUtils.LongToSpanByte(next, newValue.AsSpan()); - newValue.AsReadOnlySpan().CopyTo(output.SpanByte.AsSpan()); - output.SpanByte.Length = newValue.LengthWithoutMetadata; + NumUtils.LongToSpanByte(next, newValue.AsSpan(etagIgnoredOffset)); + newValue.AsReadOnlySpan(etagIgnoredOffset).CopyTo(output.SpanByte.AsSpan()); + output.SpanByte.Length = newValue.LengthWithoutMetadata - etagIgnoredOffset; + } + + static void CopyUpdateNumber(double next, ref SpanByte newValue, ref SpanByteAndMemory output, int etagIgnoredOffset) + { + NumUtils.DoubleToSpanByte(next, newValue.AsSpan(etagIgnoredOffset)); + newValue.AsReadOnlySpan(etagIgnoredOffset).CopyTo(output.SpanByte.AsSpan()); + output.SpanByte.Length = newValue.LengthWithoutMetadata - etagIgnoredOffset; } static void CopyUpdateNumber(double next, ref SpanByte newValue, ref SpanByteAndMemory output) @@ -497,12 +507,12 @@ static void CopyUpdateNumber(double next, ref SpanByte newValue, ref SpanByteAnd /// New value copying to /// Output value /// Parsed input value - static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, long input) + static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, long input, int etagIgnoredOffset) { newValue.ExtraMetadata = oldValue.ExtraMetadata; // Check if value contains a valid number - if (!IsValidNumber(oldValue.LengthWithoutMetadata, oldValue.ToPointer(), output.SpanByte.AsSpan(), out var val)) + if (!IsValidNumber(oldValue.LengthWithoutMetadata - etagIgnoredOffset, oldValue.ToPointer() + etagIgnoredOffset, output.SpanByte.AsSpan(), out var val)) { // Move to tail of the log even when oldValue is alphanumeric // We have already paid the cost of bringing from disk so we are treating as a regular access and bring it into memory @@ -522,7 +532,7 @@ static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, re } // Move to tail of the log and update - CopyUpdateNumber(val, ref newValue, ref output); + CopyUpdateNumber(val, ref newValue, ref output, etagIgnoredOffset); } /// @@ -532,12 +542,13 @@ static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, re /// New value copying to /// Output value /// Parsed input value - static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, double input) + /// Number of bytes to skip for ignoring etag in value payload + static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, double input, int etagIgnoredOffset) { newValue.ExtraMetadata = oldValue.ExtraMetadata; // Check if value contains a valid number - if (!IsValidDouble(oldValue.LengthWithoutMetadata, oldValue.ToPointer(), output.SpanByte.AsSpan(), out var val)) + if (!IsValidDouble(oldValue.LengthWithoutMetadata - etagIgnoredOffset, oldValue.ToPointer() + etagIgnoredOffset, output.SpanByte.AsSpan(), out var val)) { // Move to tail of the log even when oldValue is alphanumeric // We have already paid the cost of bringing from disk so we are treating as a regular access and bring it into memory @@ -553,7 +564,7 @@ static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, re } // Move to tail of the log and update - CopyUpdateNumber(val, ref newValue, ref output); + CopyUpdateNumber(val, ref newValue, ref output, etagIgnoredOffset); } /// @@ -650,16 +661,61 @@ void CopyRespNumber(long number, ref SpanByteAndMemory dst) /// /// Copy length of value to output (as ASCII bytes) /// - static void CopyValueLengthToOutput(ref SpanByte value, ref SpanByteAndMemory output) + static void CopyValueLengthToOutput(ref SpanByte value, ref SpanByteAndMemory output, int eTagIgnoredOffset) { - int numDigits = NumUtils.NumDigits(value.LengthWithoutMetadata); + int numDigits = NumUtils.NumDigits(value.LengthWithoutMetadata - eTagIgnoredOffset); Debug.Assert(output.IsSpanByte, "This code assumes it is called in a non-pending context or in a pending context where dst.SpanByte's pointer remains valid"); var outputPtr = output.SpanByte.ToPointer(); - NumUtils.IntToBytes(value.LengthWithoutMetadata, numDigits, ref outputPtr); + NumUtils.IntToBytes(value.LengthWithoutMetadata - eTagIgnoredOffset, numDigits, ref outputPtr); output.SpanByte.Length = numDigits; } + static void CopyRespWithEtagData(ref SpanByte value, ref SpanByteAndMemory dst, bool hasEtagInVal, int etagSkippedStart, MemoryPool memoryPool) + { + int valueLength = value.LengthWithoutMetadata; + // always writing an array of size 2 => *2\r\n + int desiredLength = 4; + ReadOnlySpan etagTruncatedVal; + // get etag to write, default etag 0 for when no etag + long etag = hasEtagInVal ? value.GetEtagInPayload() : Constants.BaseEtag; + // remove the length of the ETAG + var etagAccountedValueLength = valueLength - etagSkippedStart; + if (hasEtagInVal) + { + etagAccountedValueLength = valueLength - Constants.EtagSize; + } + + // here we know the value span has first bytes set to etag so we hardcode skipping past the bytes for the etag below + etagTruncatedVal = value.AsReadOnlySpan(etagSkippedStart); + // *2\r\n :(etag digits)\r\n $(val Len digits)\r\n (value len)\r\n + desiredLength += 1 + NumUtils.NumDigitsInLong(etag) + 2 + 1 + NumUtils.NumDigits(etagAccountedValueLength) + 2 + etagAccountedValueLength + 2; + + WriteValAndEtagToDst(desiredLength, ref etagTruncatedVal, etag, ref dst, memoryPool); + } + + static void WriteValAndEtagToDst(int desiredLength, ref ReadOnlySpan value, long etag, ref SpanByteAndMemory dst, MemoryPool memoryPool, bool writeDirect = false) + { + if (desiredLength <= dst.Length) + { + dst.Length = desiredLength; + byte* curr = dst.SpanByte.ToPointer(); + byte* end = curr + dst.SpanByte.Length; + RespWriteUtils.WriteEtagValArray(etag, ref value, ref curr, end, writeDirect); + return; + } + + dst.ConvertToHeap(); + dst.Length = desiredLength; + dst.Memory = memoryPool.Rent(desiredLength); + fixed (byte* ptr = dst.Memory.Memory.Span) + { + byte* curr = ptr; + byte* end = ptr + desiredLength; + RespWriteUtils.WriteEtagValArray(etag, ref value, ref curr, end, writeDirect); + } + } + /// /// Logging upsert from /// a. ConcurrentWriter diff --git a/libs/server/Storage/Functions/MainStore/RMWMethods.cs b/libs/server/Storage/Functions/MainStore/RMWMethods.cs index 9d875ac357..3ae61a4006 100644 --- a/libs/server/Storage/Functions/MainStore/RMWMethods.cs +++ b/libs/server/Storage/Functions/MainStore/RMWMethods.cs @@ -20,7 +20,6 @@ public bool NeedInitialUpdate(ref SpanByte key, ref RawStringInput input, ref Sp switch (input.header.cmd) { case RespCommand.SETKEEPTTLXX: - case RespCommand.SETEXXX: case RespCommand.PERSIST: case RespCommand.EXPIRE: case RespCommand.PEXPIRE: @@ -29,6 +28,23 @@ public bool NeedInitialUpdate(ref SpanByte key, ref RawStringInput input, ref Sp case RespCommand.GETDEL: case RespCommand.GETEX: return false; + case RespCommand.SETIFMATCH: + case RespCommand.SETEXXX: + // when called withetag all output needs to be placed on the buffer + if (input.header.CheckWithEtagFlag()) + { + // XX when unsuccesful will write back NIL + CopyDefaultResp(CmdStrings.RESP_ERRNOTFOUND, ref output); + } + return false; + case RespCommand.SET: + case RespCommand.SETEXNX: + case RespCommand.SETKEEPTTL: + if (input.header.CheckWithEtagFlag()) + { + this.functionsState.etagState.etagOffsetForVarlen = Constants.EtagSize; + } + return true; default: if (input.header.cmd > RespCommandExtensions.LastValidCommand) { @@ -39,6 +55,7 @@ public bool NeedInitialUpdate(ref SpanByte key, ref RawStringInput input, ref Sp output.Length = outp.Length; return ret; } + return true; } } @@ -48,7 +65,8 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB { rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - switch (input.header.cmd) + RespCommand cmd = input.header.cmd; + switch (cmd) { case RespCommand.PFADD: var v = value.ToPointer(); @@ -72,20 +90,42 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB case RespCommand.SET: case RespCommand.SETEXNX: + int spaceForEtag = this.functionsState.etagState.etagOffsetForVarlen; // Copy input to value var newInputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; var metadataSize = input.arg1 == 0 ? 0 : sizeof(long); value.UnmarkExtraMetadata(); - value.ShrinkSerializedLength(newInputValue.Length + metadataSize); + value.ShrinkSerializedLength(newInputValue.Length + metadataSize + spaceForEtag); value.ExtraMetadata = input.arg1; - newInputValue.CopyTo(value.AsSpan()); - break; + newInputValue.CopyTo(value.AsSpan(spaceForEtag)); + + if (spaceForEtag != 0) + { + recordInfo.SetHasETag(); + // the increment on initial etag is for satisfying the variant that any key with no etag is the same as a zero'd etag + value.SetEtagInPayload(Constants.BaseEtag + 1); + EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref value); + // Copy initial etag to output only for SET + WITHETAG and not SET NX or XX + CopyRespNumber(Constants.BaseEtag + 1, ref output); + } + break; case RespCommand.SETKEEPTTL: + spaceForEtag = this.functionsState.etagState.etagOffsetForVarlen; // Copy input to value, retain metadata in value var setValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; - value.ShrinkSerializedLength(value.MetadataSize + setValue.Length); - setValue.CopyTo(value.AsSpan()); + value.ShrinkSerializedLength(value.MetadataSize + setValue.Length + spaceForEtag); + setValue.CopyTo(value.AsSpan(spaceForEtag)); + + if (spaceForEtag != 0) + { + recordInfo.SetHasETag(); + value.SetEtagInPayload(Constants.BaseEtag + 1); + EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref value); + // Copy initial etag to output + CopyRespNumber(Constants.BaseEtag + 1, ref output); + } + break; case RespCommand.SETKEEPTTLXX: @@ -127,7 +167,7 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB var newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; newValue.CopyTo(value.AsSpan().Slice(offset)); - CopyValueLengthToOutput(ref value, ref output); + CopyValueLengthToOutput(ref value, ref output, 0); break; case RespCommand.APPEND: @@ -136,7 +176,7 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB // Copy value to be appended to the newly allocated value buffer appendValue.ReadOnlySpan.CopyTo(value.AsSpan()); - CopyValueLengthToOutput(ref value, ref output); + CopyValueLengthToOutput(ref value, ref output, 0); break; case RespCommand.INCR: value.UnmarkExtraMetadata(); @@ -176,7 +216,6 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB break; default: value.UnmarkExtraMetadata(); - if (input.header.cmd > RespCommandExtensions.LastValidCommand) { var functions = functionsState.GetCustomCommandFunctions((ushort)input.header.cmd); @@ -218,6 +257,10 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB /// public void PostInitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { + // reset etag state set at need initial update + if (input.header.cmd is (RespCommand.SET or RespCommand.SETEXNX or RespCommand.SETKEEPTTL)) + EtagState.ResetState(ref functionsState.etagState); + functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash); if (functionsState.appendOnlyFile != null) { @@ -247,69 +290,222 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re if (value.MetadataSize > 0 && input.header.CheckExpiry(value.ExtraMetadata)) { rmwInfo.Action = RMWAction.ExpireAndResume; + recordInfo.ClearHasETag(); return false; } - // First byte of input payload identifies command - switch (input.header.cmd) + RespCommand cmd = input.header.cmd; + bool hadRecordPreMutation = recordInfo.ETag; + bool shouldUpdateEtag = hadRecordPreMutation; + if (shouldUpdateEtag) + { + EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref value); + } + + switch (cmd) { case RespCommand.SETEXNX: - // Check if SetGet flag is set if (input.header.CheckSetGetFlag()) { // Copy value to output for the GET part of the command. CopyRespTo(ref value, ref output); } + else if (input.header.CheckWithEtagFlag()) + { + // when called withetag all output needs to be placed on the buffer + // EXX when unsuccesful will write back NIL + CopyDefaultResp(CmdStrings.RESP_ERRNOTFOUND, ref output); + } + + // reset etag state after done using + EtagState.ResetState(ref functionsState.etagState); + // Nothing is set because being in this block means NX was already violated return true; + case RespCommand.SETIFMATCH: + long etagFromClient = input.parseState.GetLong(1); + if (functionsState.etagState.etag != etagFromClient) + { + CopyRespWithEtagData(ref value, ref output, shouldUpdateEtag, functionsState.etagState.etagSkippedStart, functionsState.memoryPool); + // reset etag state after done using + EtagState.ResetState(ref functionsState.etagState); + return true; + } + + // Need Copy update if no space for new value + var inputValue = input.parseState.GetArgSliceByRef(0); + + // retain metadata unless metadata sent + int metadataSize = input.arg1 != 0 ? sizeof(long) : value.MetadataSize; + + if (value.Length < inputValue.length + Constants.EtagSize + metadataSize) + return false; + if (input.arg1 != 0) + { + value.ExtraMetadata = input.arg1; + } + + recordInfo.SetHasETag(); + + // Increment the ETag + long newEtag = functionsState.etagState.etag + 1; + + value.ShrinkSerializedLength(metadataSize + inputValue.Length + Constants.EtagSize); + rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); + rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); + + value.SetEtagInPayload(newEtag); + + inputValue.ReadOnlySpan.CopyTo(value.AsSpan(Constants.EtagSize)); + + // write back array of the format [etag, nil] + var nilResp = CmdStrings.RESP_ERRNOTFOUND; + // *2\r\n: + + \r\n + + var numDigitsInEtag = NumUtils.NumDigitsInLong(newEtag); + WriteValAndEtagToDst(4 + 1 + numDigitsInEtag + 2 + nilResp.Length, ref nilResp, newEtag, ref output, functionsState.memoryPool, writeDirect: true); + // reset etag state after done using + EtagState.ResetState(ref functionsState.etagState); + // early return since we already updated the ETag + return true; case RespCommand.SET: case RespCommand.SETEXXX: - var setValue = input.parseState.GetArgSliceByRef(0); + // If the user calls withetag then we need to either update an existing etag and set the value or set the value with an etag and increment it. + bool inputHeaderHasEtag = input.header.CheckWithEtagFlag(); + + int nextUpdateEtagOffset = functionsState.etagState.etagSkippedStart; + + // only when both are not false && false or true and true, do we need to readjust + if (inputHeaderHasEtag != shouldUpdateEtag) + { + // in the common path the above condition is skipped + if (inputHeaderHasEtag) + { + // nextUpdate will add etag but currently there is no etag + nextUpdateEtagOffset = Constants.EtagSize; + shouldUpdateEtag = true; + // if something is going to go past this into copy we need to provide offset management for its varlen during allocation + this.functionsState.etagState.etagOffsetForVarlen = Constants.EtagSize; + } + else + { + shouldUpdateEtag = false; + // nextUpdate will remove etag but currently there is an etag + nextUpdateEtagOffset = 0; + this.functionsState.etagState.etagOffsetForVarlen = 0; + } + } + + ArgSlice setValue = input.parseState.GetArgSliceByRef(0); // Need CU if no space for new value - var metadataSize = input.arg1 == 0 ? 0 : sizeof(long); - if (setValue.Length + metadataSize > value.Length) return false; + metadataSize = input.arg1 == 0 ? 0 : sizeof(long); + if (setValue.Length + metadataSize > value.Length - nextUpdateEtagOffset) + return false; // Check if SetGet flag is set if (input.header.CheckSetGetFlag()) { + Debug.Assert(!input.header.CheckWithEtagFlag(), "SET GET CANNNOT BE CALLED WITH WITHETAG"); // Copy value to output for the GET part of the command. - CopyRespTo(ref value, ref output); + CopyRespTo(ref value, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); } // Adjust value length rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); value.UnmarkExtraMetadata(); - value.ShrinkSerializedLength(setValue.Length + metadataSize); + value.ShrinkSerializedLength(setValue.Length + metadataSize + nextUpdateEtagOffset); // Copy input to value value.ExtraMetadata = input.arg1; - setValue.ReadOnlySpan.CopyTo(value.AsSpan()); + setValue.ReadOnlySpan.CopyTo(value.AsSpan(nextUpdateEtagOffset)); rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); - return true; + // If withEtag is called we return the etag back in the response + if (inputHeaderHasEtag) + { + recordInfo.SetHasETag(); + value.SetEtagInPayload(functionsState.etagState.etag + 1); + // withetag flag means we need to write etag back to the output buffer + CopyRespNumber(functionsState.etagState.etag + 1, ref output); + // reset etag state after done using + EtagState.ResetState(ref functionsState.etagState); + // early return since we already updated etag + return true; + } + else + { + recordInfo.ClearHasETag(); + } + break; case RespCommand.SETKEEPTTLXX: case RespCommand.SETKEEPTTL: + // If the user calls withetag then we need to either update an existing etag and set the value + // or set the value with an initial etag and increment it. + // If withEtag is called we return the etag back to the user + inputHeaderHasEtag = input.header.CheckWithEtagFlag(); + + nextUpdateEtagOffset = functionsState.etagState.etagSkippedStart; + + // only when both are not false && false or true and true, do we need to readjust + if (inputHeaderHasEtag != shouldUpdateEtag) + { + // in the common path the above condition is skipped + if (inputHeaderHasEtag) + { + // nextUpdate will add etag but currently there is no etag + nextUpdateEtagOffset = Constants.EtagSize; + shouldUpdateEtag = true; + // if something is going to go past this into copy we need to provide offset management for its varlen during allocation + this.functionsState.etagState.etagOffsetForVarlen = Constants.EtagSize; + } + else + { + shouldUpdateEtag = false; + // nextUpdate will remove etag but currentyly there is an etag + nextUpdateEtagOffset = 0; + this.functionsState.etagState.etagOffsetForVarlen = 0; + } + } + setValue = input.parseState.GetArgSliceByRef(0); // Need CU if no space for new value - if (setValue.Length + value.MetadataSize > value.Length) return false; + if (setValue.Length + value.MetadataSize > value.Length - nextUpdateEtagOffset) + return false; // Check if SetGet flag is set if (input.header.CheckSetGetFlag()) { + Debug.Assert(!input.header.CheckWithEtagFlag(), "SET GET CANNNOT BE CALLED WITH WITHETAG"); // Copy value to output for the GET part of the command. - CopyRespTo(ref value, ref output); + CopyRespTo(ref value, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); } // Adjust value length rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.ShrinkSerializedLength(setValue.Length + value.MetadataSize); + value.ShrinkSerializedLength(setValue.Length + value.MetadataSize + functionsState.etagState.etagSkippedStart); // Copy input to value - setValue.ReadOnlySpan.CopyTo(value.AsSpan()); + setValue.ReadOnlySpan.CopyTo(value.AsSpan(functionsState.etagState.etagSkippedStart)); rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); - return true; + + if (inputHeaderHasEtag) + { + recordInfo.SetHasETag(); + value.SetEtagInPayload(functionsState.etagState.etag + 1); + // withetag flag means we need to write etag back to the output buffer + CopyRespNumber(functionsState.etagState.etag + 1, ref output); + // reset etag state after done using + EtagState.ResetState(ref functionsState.etagState); + // early return since we already updated etag + return true; + } + else + { + recordInfo.ClearHasETag(); + } + + break; case RespCommand.PEXPIRE: case RespCommand.EXPIRE: @@ -322,8 +518,14 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re var expiryTicks = DateTimeOffset.UtcNow.Ticks + tsExpiry.Ticks; var expireOption = (ExpireOption)input.arg1; - return EvaluateExpireInPlace(expireOption, expiryExists, expiryTicks, ref value, ref output); + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); + + if (!EvaluateExpireInPlace(expireOption, expiryExists, expiryTicks, ref value, ref output)) + return false; + // doesn't update etag, since it's only the metadata that was updated + return true; ; case RespCommand.PEXPIREAT: case RespCommand.EXPIREAT: expiryExists = value.MetadataSize > 0; @@ -334,7 +536,14 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re : ConvertUtils.UnixTimestampInSecondsToTicks(expiryTimestamp); expireOption = (ExpireOption)input.arg1; - return EvaluateExpireInPlace(expireOption, expiryExists, expiryTicks, ref value, ref output); + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); + + if (!EvaluateExpireInPlace(expireOption, expiryExists, expiryTicks, ref value, ref output)) + return false; + + // doesn't update etag, since it's only the metadata that was updated + return true; case RespCommand.PERSIST: if (value.MetadataSize != 0) @@ -346,38 +555,56 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); output.SpanByte.AsSpan()[0] = 1; } + // does not update etag + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); return true; case RespCommand.INCR: - return TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: 1); + if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: 1, functionsState.etagState.etagSkippedStart)) + return false; + break; case RespCommand.DECR: - return TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: -1); + if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: -1, functionsState.etagState.etagSkippedStart)) + { + + return false; + } + break; case RespCommand.INCRBY: // Check if input contains a valid number var incrBy = input.arg1; - return TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: incrBy); + if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: incrBy, functionsState.etagState.etagSkippedStart)) + return false; + break; case RespCommand.DECRBY: var decrBy = input.arg1; - return TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: -decrBy); + if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: -decrBy, functionsState.etagState.etagSkippedStart)) + return false; + break; case RespCommand.INCRBYFLOAT: // Check if input contains a valid number if (!input.parseState.TryGetDouble(0, out var incrByFloat)) { output.SpanByte.AsSpan()[0] = (byte)OperationError.INVALID_TYPE; + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); return true; } - return TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, incrByFloat); + if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, incrByFloat, functionsState.etagState.etagSkippedStart)) + return false; + break; case RespCommand.SETBIT: - var v = value.ToPointer(); + var v = value.ToPointer() + functionsState.etagState.etagSkippedStart; var bOffset = input.parseState.GetLong(0); var bSetVal = (byte)(input.parseState.GetArgSliceByRef(1).ReadOnlySpan[0] - '0'); - if (!BitmapManager.IsLargeEnough(value.Length, bOffset)) return false; + if (!BitmapManager.IsLargeEnough(functionsState.etagState.etagAccountedLength, bOffset)) return false; rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); value.UnmarkExtraMetadata(); @@ -389,24 +616,32 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output); else CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_1, ref output); - return true; + break; case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); - v = value.ToPointer(); - if (!BitmapManager.IsLargeEnoughForType(bitFieldArgs, value.Length)) return false; + v = value.ToPointer() + functionsState.etagState.etagSkippedStart; + + if (!BitmapManager.IsLargeEnoughForType(bitFieldArgs, value.Length - functionsState.etagState.etagSkippedStart)) + return false; rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); value.UnmarkExtraMetadata(); value.ShrinkSerializedLength(value.Length + value.MetadataSize); rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); - var (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, v, value.Length); + var (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, v, value.Length - functionsState.etagState.etagSkippedStart); - if (!overflow) - CopyRespNumber(bitfieldReturnValue, ref output); - else + if (overflow) + { CopyDefaultResp(CmdStrings.RESP_ERRNOTFOUND, ref output); - return true; + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); + // etag not updated + return true; + } + + CopyRespNumber(bitfieldReturnValue, ref output); + break; case RespCommand.PFADD: v = value.ToPointer(); @@ -414,6 +649,8 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re if (!HyperLogLog.DefaultHLL.IsValidHYLL(v, value.Length)) { *output.SpanByte.ToPointer() = (byte)0xFF; + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); return true; } @@ -434,6 +671,8 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re if (!HyperLogLog.DefaultHLL.IsValidHYLL(dstHLL, value.Length)) { + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); //InvalidType *(long*)output.SpanByte.ToPointer() = -1; return true; @@ -446,23 +685,23 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re var offset = input.parseState.GetInt(0); var newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; - if (newValue.Length + offset > value.LengthWithoutMetadata) + if (newValue.Length + offset > value.LengthWithoutMetadata - functionsState.etagState.etagSkippedStart) return false; - newValue.CopyTo(value.AsSpan().Slice(offset)); + newValue.CopyTo(value.AsSpan(functionsState.etagState.etagSkippedStart).Slice(offset)); - CopyValueLengthToOutput(ref value, ref output); - return true; + CopyValueLengthToOutput(ref value, ref output, functionsState.etagState.etagSkippedStart); + break; case RespCommand.GETDEL: // Copy value to output for the GET part of the command. // Then, set ExpireAndStop action to delete the record. - CopyRespTo(ref value, ref output); + CopyRespTo(ref value, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); rmwInfo.Action = RMWAction.ExpireAndStop; return false; case RespCommand.GETEX: - CopyRespTo(ref value, ref output); + CopyRespTo(ref value, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); if (input.arg1 > 0) { @@ -485,10 +724,14 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re value.ShrinkSerializedLength(value.Length - value.MetadataSize); value.UnmarkExtraMetadata(); rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); return true; } } + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); return true; case RespCommand.APPEND: @@ -497,16 +740,24 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re if (appendSize == 0) { - CopyValueLengthToOutput(ref value, ref output); + CopyValueLengthToOutput(ref value, ref output, functionsState.etagState.etagSkippedStart); + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); return true; } return false; - default: - var cmd = input.header.cmd; if (cmd > RespCommandExtensions.LastValidCommand) { + if (shouldUpdateEtag) + { + CopyDefaultResp(CmdStrings.RESP_ERR_ETAG_ON_CUSTOM_PROC, ref output); + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); + return true; + } + var functions = functionsState.GetCustomCommandFunctions((ushort)cmd); var expiration = input.arg1; if (expiration == -1) @@ -550,6 +801,20 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re } throw new GarnetException("Unsupported operation on input"); } + + // increment the Etag transparently if in place update happened + if (shouldUpdateEtag) + { + value.SetEtagInPayload(this.functionsState.etagState.etag + 1); + } + + if (hadRecordPreMutation) + { + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); + } + + return true; } /// @@ -557,40 +822,78 @@ public bool NeedCopyUpdate(ref SpanByte key, ref RawStringInput input, ref SpanB { switch (input.header.cmd) { + case RespCommand.SETIFMATCH: + + long etagToCheckWith = input.parseState.GetLong(1); + + if (functionsState.etagState.etag == etagToCheckWith) + { + return true; + } + else + { + CopyRespWithEtagData(ref oldValue, ref output, hasEtagInVal: rmwInfo.RecordInfo.ETag, functionsState.etagState.etagSkippedStart, functionsState.memoryPool); + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); + return false; + } + case RespCommand.SETEXNX: // Expired data, return false immediately // ExpireAndResume ensures that we set as new value, since it does not exist if (oldValue.MetadataSize > 0 && input.header.CheckExpiry(oldValue.ExtraMetadata)) { rmwInfo.Action = RMWAction.ExpireAndResume; + rmwInfo.RecordInfo.ClearHasETag(); + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); return false; } - // Check if SetGet flag is set + + // since this case is only hit when this an update, the NX is violated and so we can return early from it without setting the value + if (input.header.CheckSetGetFlag()) { // Copy value to output for the GET part of the command. - CopyRespTo(ref oldValue, ref output); + CopyRespTo(ref oldValue, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); } + else if (input.header.CheckWithEtagFlag()) + { + // EXX when unsuccesful will write back NIL + CopyDefaultResp(CmdStrings.RESP_ERRNOTFOUND, ref output); + } + + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); return false; case RespCommand.SETEXXX: // Expired data, return false immediately so we do not set, since it does not exist // ExpireAndStop ensures that caller sees a NOTFOUND status if (oldValue.MetadataSize > 0 && input.header.CheckExpiry(oldValue.ExtraMetadata)) { + rmwInfo.RecordInfo.ClearHasETag(); rmwInfo.Action = RMWAction.ExpireAndStop; + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); return false; } return true; default: if (input.header.cmd > RespCommandExtensions.LastValidCommand) { + if (rmwInfo.RecordInfo.ETag) + { + CopyDefaultResp(CmdStrings.RESP_ERR_ETAG_ON_CUSTOM_PROC, ref output); + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); + return false; + } (IMemoryOwner Memory, int Length) outp = (output.Memory, 0); var ret = functionsState.GetCustomCommandFunctions((ushort)input.header.cmd) - .NeedCopyUpdate(key.AsReadOnlySpan(), ref input, oldValue.AsReadOnlySpan(), ref outp); + .NeedCopyUpdate(key.AsReadOnlySpan(), ref input, oldValue.AsReadOnlySpan(functionsState.etagState.etagSkippedStart), ref outp); output.Memory = outp.Memory; output.Length = outp.Length; return ret; - } return true; } @@ -602,52 +905,161 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte // Expired data if (oldValue.MetadataSize > 0 && input.header.CheckExpiry(oldValue.ExtraMetadata)) { + recordInfo.ClearHasETag(); rmwInfo.Action = RMWAction.ExpireAndResume; + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); return false; } rmwInfo.ClearExtraValueLength(ref recordInfo, ref newValue, newValue.TotalSize); - switch (input.header.cmd) + RespCommand cmd = input.header.cmd; + + bool recordHadEtagPreMutation = recordInfo.ETag; + bool shouldUpdateEtag = recordHadEtagPreMutation; + if (shouldUpdateEtag) { + // during checkpointing we might skip the inplace calls and go directly to copy update so we need to initialize here if needed + EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref oldValue); + } + + switch (cmd) + { + case RespCommand.SETIFMATCH: + shouldUpdateEtag = true; + // Copy input to value + Span dest = newValue.AsSpan(Constants.EtagSize); + ReadOnlySpan src = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + + // retain metadata unless metadata sent + int metadataSize = input.arg1 != 0 ? sizeof(long) : oldValue.MetadataSize; + + Debug.Assert(src.Length + Constants.EtagSize + metadataSize == newValue.Length); + + src.CopyTo(dest); + + newValue.ExtraMetadata = oldValue.ExtraMetadata; + if (input.arg1 != 0) + { + newValue.ExtraMetadata = input.arg1; + } + + long newEtag = functionsState.etagState.etag + 1; + + recordInfo.SetHasETag(); + + // Write Etag and Val back to Client + // write back array of the format [etag, nil] + var nilResp = CmdStrings.RESP_ERRNOTFOUND; + // *2\r\n: + + \r\n + + var numDigitsInEtag = NumUtils.NumDigitsInLong(newEtag); + WriteValAndEtagToDst(4 + 1 + numDigitsInEtag + 2 + nilResp.Length, ref nilResp, newEtag, ref output, functionsState.memoryPool, writeDirect: true); + break; case RespCommand.SET: case RespCommand.SETEXXX: + var nextUpdateEtagOffset = functionsState.etagState.etagSkippedStart; + var nextUpdateEtagAccountedLength = functionsState.etagState.etagAccountedLength; + bool inputWithEtag = input.header.CheckWithEtagFlag(); + + // only when both are not false && false or true and true, do we need to readjust + if (inputWithEtag != shouldUpdateEtag) + { + // in the common path the above condition is skipped + if (inputWithEtag) + { + // nextUpdate will add etag but currently there is no etag + nextUpdateEtagOffset = Constants.EtagSize; + shouldUpdateEtag = true; + recordInfo.SetHasETag(); + } + else + { + // nextUpdate will remove etag but currentyly there is an etag + nextUpdateEtagOffset = 0; + shouldUpdateEtag = false; + recordInfo.ClearHasETag(); + } + } + // Check if SetGet flag is set if (input.header.CheckSetGetFlag()) { + Debug.Assert(!input.header.CheckWithEtagFlag(), "SET GET CANNNOT BE CALLED WITH WITHETAG"); // Copy value to output for the GET part of the command. - CopyRespTo(ref oldValue, ref output); + CopyRespTo(ref oldValue, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); } // Copy input to value var newInputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; - var metadataSize = input.arg1 == 0 ? 0 : sizeof(long); + metadataSize = input.arg1 == 0 ? 0 : sizeof(long); - Debug.Assert(newInputValue.Length + metadataSize == newValue.Length); + // new value when allocated should have 8 bytes more if the previous record had etag and the cmd was not SETEXXX + Debug.Assert(newInputValue.Length + metadataSize + nextUpdateEtagOffset == newValue.Length); newValue.ExtraMetadata = input.arg1; - newInputValue.CopyTo(newValue.AsSpan()); + newInputValue.CopyTo(newValue.AsSpan(nextUpdateEtagOffset)); + + if (inputWithEtag) + { + CopyRespNumber(functionsState.etagState.etag + 1, ref output); + } + break; case RespCommand.SETKEEPTTLXX: case RespCommand.SETKEEPTTL: + nextUpdateEtagOffset = functionsState.etagState.etagSkippedStart; + nextUpdateEtagAccountedLength = functionsState.etagState.etagAccountedLength; + inputWithEtag = input.header.CheckWithEtagFlag(); + + // only when both are not false && false or true and true, do we need to readjust + if (inputWithEtag != shouldUpdateEtag) + { + // in the common path the above condition is skipped + if (inputWithEtag) + { + // nextUpdate will add etag but currently there is no etag + nextUpdateEtagOffset = Constants.EtagSize; + shouldUpdateEtag = true; + recordInfo.SetHasETag(); + } + else + { + shouldUpdateEtag = false; + // nextUpdate will remove etag but currentyly there is an etag + nextUpdateEtagOffset = 0; + recordInfo.ClearHasETag(); + } + } + var setValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; - Debug.Assert(oldValue.MetadataSize + setValue.Length == newValue.Length); + + Debug.Assert(oldValue.MetadataSize + setValue.Length + nextUpdateEtagOffset == newValue.Length); // Check if SetGet flag is set if (input.header.CheckSetGetFlag()) { + Debug.Assert(!input.header.CheckWithEtagFlag(), "SET GET CANNNOT BE CALLED WITH WITHETAG"); // Copy value to output for the GET part of the command. - CopyRespTo(ref oldValue, ref output); + CopyRespTo(ref oldValue, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); } // Copy input to value, retain metadata of oldValue newValue.ExtraMetadata = oldValue.ExtraMetadata; - setValue.CopyTo(newValue.AsSpan()); + setValue.CopyTo(newValue.AsSpan(nextUpdateEtagOffset)); + + if (inputWithEtag) + { + CopyRespNumber(functionsState.etagState.etag + 1, ref output); + } + break; case RespCommand.EXPIRE: case RespCommand.PEXPIRE: + shouldUpdateEtag = false; + var expiryExists = oldValue.MetadataSize > 0; var expiryValue = input.parseState.GetLong(0); @@ -663,6 +1075,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte case RespCommand.PEXPIREAT: case RespCommand.EXPIREAT: expiryExists = oldValue.MetadataSize > 0; + shouldUpdateEtag = false; var expiryTimestamp = input.parseState.GetLong(0); expiryTicks = input.header.cmd == RespCommand.PEXPIREAT @@ -674,6 +1087,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte break; case RespCommand.PERSIST: + shouldUpdateEtag = false; oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); if (oldValue.MetadataSize != 0) { @@ -685,21 +1099,21 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte break; case RespCommand.INCR: - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: 1); + TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: 1, functionsState.etagState.etagSkippedStart); break; case RespCommand.DECR: - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: -1); + TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: -1, functionsState.etagState.etagSkippedStart); break; case RespCommand.INCRBY: var incrBy = input.arg1; - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: incrBy); + TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: incrBy, functionsState.etagState.etagSkippedStart); break; case RespCommand.DECRBY: var decrBy = input.arg1; - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: -decrBy); + TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: -decrBy, functionsState.etagState.etagSkippedStart); break; case RespCommand.INCRBYFLOAT: @@ -710,7 +1124,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte oldValue.CopyTo(ref newValue); break; } - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: incrByFloat); + TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: incrByFloat, functionsState.etagState.etagSkippedStart); break; case RespCommand.SETBIT: @@ -726,8 +1140,8 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); - Buffer.MemoryCopy(oldValue.ToPointer(), newValue.ToPointer(), newValue.Length, oldValue.Length); - var (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, newValue.ToPointer(), newValue.Length); + Buffer.MemoryCopy(oldValue.ToPointer() + functionsState.etagState.etagSkippedStart, newValue.ToPointer() + functionsState.etagState.etagSkippedStart, newValue.Length - functionsState.etagState.etagSkippedStart, oldValue.Length - functionsState.etagState.etagSkippedStart); + var (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, newValue.ToPointer() + functionsState.etagState.etagSkippedStart, newValue.Length - functionsState.etagState.etagSkippedStart); if (!overflow) CopyRespNumber(bitfieldReturnValue, ref output); @@ -764,20 +1178,24 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte oldValue.CopyTo(ref newValue); newInputValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; - newInputValue.CopyTo(newValue.AsSpan().Slice(offset)); + newInputValue.CopyTo(newValue.AsSpan(functionsState.etagState.etagSkippedStart).Slice(offset)); - CopyValueLengthToOutput(ref newValue, ref output); + CopyValueLengthToOutput(ref newValue, ref output, functionsState.etagState.etagSkippedStart); break; case RespCommand.GETDEL: // Copy value to output for the GET part of the command. // Then, set ExpireAndStop action to delete the record. - CopyRespTo(ref oldValue, ref output); + CopyRespTo(ref oldValue, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); rmwInfo.Action = RMWAction.ExpireAndStop; + + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); return false; case RespCommand.GETEX: - CopyRespTo(ref oldValue, ref output); + shouldUpdateEtag = false; + CopyRespTo(ref oldValue, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); if (input.arg1 > 0) { @@ -813,12 +1231,20 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte // Append the new value with the client input at the end of the old data appendValue.ReadOnlySpan.CopyTo(newValue.AsSpan().Slice(oldValue.LengthWithoutMetadata)); - CopyValueLengthToOutput(ref newValue, ref output); + CopyValueLengthToOutput(ref newValue, ref output, functionsState.etagState.etagSkippedStart); break; default: if (input.header.cmd > RespCommandExtensions.LastValidCommand) { + if (recordInfo.ETag) + { + CopyDefaultResp(CmdStrings.RESP_ERR_ETAG_ON_CUSTOM_PROC, ref output); + // reset etag state that may have been initialized earlier + EtagState.ResetState(ref functionsState.etagState); + return true; + } + var functions = functionsState.GetCustomCommandFunctions((ushort)input.header.cmd); var expiration = input.arg1; if (expiration == 0) @@ -835,7 +1261,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte (IMemoryOwner Memory, int Length) outp = (output.Memory, 0); var ret = functions - .CopyUpdater(key.AsReadOnlySpan(), ref input, oldValue.AsReadOnlySpan(), newValue.AsSpan(), ref outp, ref rmwInfo); + .CopyUpdater(key.AsReadOnlySpan(), ref input, oldValue.AsReadOnlySpan(functionsState.etagState.etagSkippedStart), newValue.AsSpan(functionsState.etagState.etagSkippedStart), ref outp, ref rmwInfo); output.Memory = outp.Memory; output.Length = outp.Length; return ret; @@ -844,6 +1270,15 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte } rmwInfo.SetUsedValueLength(ref recordInfo, ref newValue, newValue.TotalSize); + + if (shouldUpdateEtag) + { + newValue.SetEtagInPayload(functionsState.etagState.etag + 1); + EtagState.ResetState(ref functionsState.etagState); + } + else if (recordHadEtagPreMutation) + EtagState.ResetState(ref functionsState.etagState); + return true; } diff --git a/libs/server/Storage/Functions/MainStore/ReadMethods.cs b/libs/server/Storage/Functions/MainStore/ReadMethods.cs index 5447708a09..cbdf3f8776 100644 --- a/libs/server/Storage/Functions/MainStore/ReadMethods.cs +++ b/libs/server/Storage/Functions/MainStore/ReadMethods.cs @@ -3,6 +3,8 @@ using System.Buffers; using System.Diagnostics; +using System.Runtime.CompilerServices; +using Garnet.common; using Tsavorite.core; namespace Garnet.server @@ -13,14 +15,31 @@ namespace Garnet.server public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions { /// - public bool SingleReader(ref SpanByte key, ref RawStringInput input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo) + public bool SingleReader( + ref SpanByte key, ref RawStringInput input, + ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo) { if (value.MetadataSize != 0 && CheckExpiry(ref value)) + { + readInfo.RecordInfo.ClearHasETag(); return false; + } var cmd = input.header.cmd; - if (cmd > RespCommandExtensions.LastValidCommand) + + if (cmd == RespCommand.GETIFNOTMATCH) + { + if (handleGetIfNotMatch(ref input, ref value, ref dst, ref readInfo)) + return true; + } + else if (cmd > RespCommandExtensions.LastValidCommand) { + if (readInfo.RecordInfo.ETag) + { + CopyDefaultResp(CmdStrings.RESP_ERR_ETAG_ON_CUSTOM_PROC, ref dst); + return true; + } + var valueLength = value.LengthWithoutMetadata; (IMemoryOwner Memory, int Length) output = (dst.Memory, 0); var ret = functionsState.GetCustomCommandFunctions((ushort)cmd) @@ -31,27 +50,60 @@ public bool SingleReader(ref SpanByte key, ref RawStringInput input, ref SpanByt return ret; } + if (readInfo.RecordInfo.ETag) + { + EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref value); + } + + // Unless the command explicitly asks for the ETag in response, we do not write back the ETag + if (cmd is (RespCommand.GETWITHETAG or RespCommand.GETIFNOTMATCH)) + { + CopyRespWithEtagData(ref value, ref dst, readInfo.RecordInfo.ETag, functionsState.etagState.etagSkippedStart, functionsState.memoryPool); + EtagState.ResetState(ref functionsState.etagState); + return true; + } + if (cmd == RespCommand.NONE) - CopyRespTo(ref value, ref dst); + CopyRespTo(ref value, ref dst, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); else + { CopyRespToWithInput(ref input, ref value, ref dst, readInfo.IsFromPending); + } + + if (readInfo.RecordInfo.ETag) + { + EtagState.ResetState(ref functionsState.etagState); + } return true; } /// - public bool ConcurrentReader(ref SpanByte key, ref RawStringInput input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) + public bool ConcurrentReader( + ref SpanByte key, ref RawStringInput input, ref SpanByte value, + ref SpanByteAndMemory dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) { if (value.MetadataSize != 0 && CheckExpiry(ref value)) { - // TODO: we can proactively expire if we wish, but if we do, we need to write WAL entry - // readInfo.Action = ReadAction.Expire; + recordInfo.ClearHasETag(); return false; } var cmd = input.header.cmd; - if (cmd > RespCommandExtensions.LastValidCommand) + + if (cmd == RespCommand.GETIFNOTMATCH) + { + if (handleGetIfNotMatch(ref input, ref value, ref dst, ref readInfo)) + return true; + } + else if (cmd > RespCommandExtensions.LastValidCommand) { + if (readInfo.RecordInfo.ETag) + { + CopyDefaultResp(CmdStrings.RESP_ERR_ETAG_ON_CUSTOM_PROC, ref dst); + return true; + } + var valueLength = value.LengthWithoutMetadata; (IMemoryOwner Memory, int Length) output = (dst.Memory, 0); var ret = functionsState.GetCustomCommandFunctions((ushort)cmd) @@ -62,14 +114,53 @@ public bool ConcurrentReader(ref SpanByte key, ref RawStringInput input, ref Spa return ret; } + if (readInfo.RecordInfo.ETag) + { + EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref value); + } + + // Unless the command explicitly asks for the ETag in response, we do not write back the ETag + if (cmd is (RespCommand.GETWITHETAG or RespCommand.GETIFNOTMATCH)) + { + CopyRespWithEtagData(ref value, ref dst, readInfo.RecordInfo.ETag, functionsState.etagState.etagSkippedStart, functionsState.memoryPool); + EtagState.ResetState(ref functionsState.etagState); + return true; + } + + if (cmd == RespCommand.NONE) - CopyRespTo(ref value, ref dst); + CopyRespTo(ref value, ref dst, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); else { CopyRespToWithInput(ref input, ref value, ref dst, readInfo.IsFromPending); } + if (readInfo.RecordInfo.ETag) + { + EtagState.ResetState(ref functionsState.etagState); + } + return true; } + + private bool handleGetIfNotMatch(ref RawStringInput input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo) + { + // Any value without an etag is treated the same as a value with an etag + long etagToMatchAgainst = input.parseState.GetLong(0); + + long existingEtag = readInfo.RecordInfo.ETag ? value.GetEtagInPayload() : Constants.BaseEtag; + + if (existingEtag == etagToMatchAgainst) + { + // write back array of the format [etag, nil] + var nilResp = CmdStrings.RESP_ERRNOTFOUND; + // *2\r\n: + + \r\n + + var numDigitsInEtag = NumUtils.NumDigitsInLong(existingEtag); + WriteValAndEtagToDst(4 + 1 + numDigitsInEtag + 2 + nilResp.Length, ref nilResp, existingEtag, ref dst, functionsState.memoryPool, writeDirect: true); + return true; + } + + return false; + } } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/MainStore/UpsertMethods.cs b/libs/server/Storage/Functions/MainStore/UpsertMethods.cs index 80f5cb6127..4bf44de3b3 100644 --- a/libs/server/Storage/Functions/MainStore/UpsertMethods.cs +++ b/libs/server/Storage/Functions/MainStore/UpsertMethods.cs @@ -12,7 +12,11 @@ namespace Garnet.server { /// public bool SingleWriter(ref SpanByte key, ref RawStringInput input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - => SpanByteFunctions.DoSafeCopy(ref src, ref dst, ref upsertInfo, ref recordInfo, input.arg1); + { + // Since upsert may be on existing key we need to wipe out the record info property + recordInfo.ClearHasETag(); + return SpanByteFunctions.DoSafeCopy(ref src, ref dst, ref upsertInfo, ref recordInfo, input.arg1); + } /// public void PostSingleWriter(ref SpanByte key, ref RawStringInput input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, WriteReason reason) @@ -25,6 +29,8 @@ public void PostSingleWriter(ref SpanByte key, ref RawStringInput input, ref Spa /// public bool ConcurrentWriter(ref SpanByte key, ref RawStringInput input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) { + // Since upsert may be on existing key we need to wipe out the record info property + recordInfo.ClearHasETag(); if (ConcurrentWriterWorker(ref src, ref dst, ref input, ref upsertInfo, ref recordInfo)) { if (!upsertInfo.RecordInfo.Modified) diff --git a/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs b/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs index 442cf7a769..3d836eaf12 100644 --- a/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs +++ b/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System.Diagnostics; using Garnet.common; using Tsavorite.core; @@ -70,6 +69,7 @@ static bool IsValidDouble(int length, byte* source, out double val) public int GetRMWInitialValueLength(ref RawStringInput input) { var cmd = input.header.cmd; + switch (cmd) { case RespCommand.SETBIT: @@ -109,7 +109,6 @@ public int GetRMWInitialValueLength(ref RawStringInput input) ndigits = NumUtils.NumDigitsInLong(-input.arg1, ref fNeg); return sizeof(int) + ndigits + (fNeg ? 1 : 0); - case RespCommand.INCRBYFLOAT: if (!input.parseState.TryGetDouble(0, out var incrByFloat)) return sizeof(int); @@ -117,7 +116,6 @@ public int GetRMWInitialValueLength(ref RawStringInput input) ndigits = NumUtils.NumOfCharInDouble(incrByFloat, out var _, out var _, out var _); return sizeof(int) + ndigits; - default: if (cmd > RespCommandExtensions.LastValidCommand) { @@ -132,8 +130,7 @@ public int GetRMWInitialValueLength(ref RawStringInput input) return sizeof(int) + metadataSize + functions.GetInitialLength(ref input); } - return sizeof(int) + input.parseState.GetArgSliceByRef(0).ReadOnlySpan.Length + - (input.arg1 == 0 ? 0 : sizeof(long)); + return sizeof(int) + input.parseState.GetArgSliceByRef(0).ReadOnlySpan.Length + (input.arg1 == 0 ? 0 : sizeof(long)) + this.functionsState.etagState.etagOffsetForVarlen; } } @@ -143,43 +140,44 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) if (input.header.cmd != RespCommand.NONE) { var cmd = input.header.cmd; + switch (cmd) { case RespCommand.INCR: case RespCommand.INCRBY: var incrByValue = input.header.cmd == RespCommand.INCRBY ? input.arg1 : 1; - var curr = NumUtils.BytesToLong(t.AsSpan()); + var curr = NumUtils.BytesToLong(t.AsSpan(functionsState.etagState.etagOffsetForVarlen)); var next = curr + incrByValue; var fNeg = false; var ndigits = NumUtils.NumDigitsInLong(next, ref fNeg); ndigits += fNeg ? 1 : 0; - return sizeof(int) + ndigits + t.MetadataSize; + return sizeof(int) + ndigits + t.MetadataSize + functionsState.etagState.etagOffsetForVarlen; case RespCommand.DECR: case RespCommand.DECRBY: var decrByValue = input.header.cmd == RespCommand.DECRBY ? input.arg1 : 1; - curr = NumUtils.BytesToLong(t.AsSpan()); + curr = NumUtils.BytesToLong(t.AsSpan(functionsState.etagState.etagOffsetForVarlen)); next = curr - decrByValue; fNeg = false; ndigits = NumUtils.NumDigitsInLong(next, ref fNeg); ndigits += fNeg ? 1 : 0; - return sizeof(int) + ndigits + t.MetadataSize; + return sizeof(int) + ndigits + t.MetadataSize + functionsState.etagState.etagOffsetForVarlen; case RespCommand.INCRBYFLOAT: // We don't need to TryGetDouble here because InPlaceUpdater will raise an error before we reach this point var incrByFloat = input.parseState.GetDouble(0); - NumUtils.TryBytesToDouble(t.AsSpan(), out var currVal); + NumUtils.TryBytesToDouble(t.AsSpan(functionsState.etagState.etagOffsetForVarlen), out var currVal); var nextVal = currVal + incrByFloat; ndigits = NumUtils.NumOfCharInDouble(nextVal, out _, out _, out _); - return sizeof(int) + ndigits + t.MetadataSize; + return sizeof(int) + ndigits + t.MetadataSize + functionsState.etagState.etagOffsetForVarlen; case RespCommand.SETBIT: var bOffset = input.parseState.GetLong(0); return sizeof(int) + BitmapManager.NewBlockAllocLength(t.Length, bOffset); @@ -202,14 +200,17 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) case RespCommand.SETKEEPTTLXX: case RespCommand.SETKEEPTTL: var setValue = input.parseState.GetArgSliceByRef(0); - return sizeof(int) + t.MetadataSize + setValue.Length; + return sizeof(int) + t.MetadataSize + setValue.Length + functionsState.etagState.etagOffsetForVarlen; case RespCommand.SET: case RespCommand.SETEXXX: - break; + return sizeof(int) + input.parseState.GetArgSliceByRef(0).Length + (input.arg1 == 0 ? 0 : sizeof(long)) + functionsState.etagState.etagOffsetForVarlen; case RespCommand.PERSIST: return sizeof(int) + t.LengthWithoutMetadata; - + case RespCommand.SETIFMATCH: + var newValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + int metadataSize = input.arg1 == 0 ? t.MetadataSize : sizeof(long); + return sizeof(int) + newValue.Length + Constants.EtagSize + metadataSize; case RespCommand.EXPIRE: case RespCommand.PEXPIRE: case RespCommand.EXPIREAT: @@ -218,10 +219,10 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) case RespCommand.SETRANGE: var offset = input.parseState.GetInt(0); - var newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; + newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; - if (newValue.Length + offset > t.LengthWithoutMetadata) - return sizeof(int) + newValue.Length + offset + t.MetadataSize; + if (newValue.Length + offset > t.LengthWithoutMetadata - functionsState.etagState.etagOffsetForVarlen) + return sizeof(int) + newValue.Length + offset + t.MetadataSize + functionsState.etagState.etagOffsetForVarlen; return sizeof(int) + t.Length; case RespCommand.GETDEL: @@ -240,7 +241,7 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) { var functions = functionsState.GetCustomCommandFunctions((ushort)cmd); // compute metadata for result - var metadataSize = input.arg1 switch + metadataSize = input.arg1 switch { -1 => 0, 0 => t.MetadataSize, diff --git a/libs/server/Storage/Session/MainStore/MainStoreOps.cs b/libs/server/Storage/Session/MainStore/MainStoreOps.cs index c1e069ba43..60d04f3f0e 100644 --- a/libs/server/Storage/Session/MainStore/MainStoreOps.cs +++ b/libs/server/Storage/Session/MainStore/MainStoreOps.cs @@ -550,9 +550,9 @@ public GarnetStatus DELETE(byte[] key, StoreType store return found ? GarnetStatus.OK : GarnetStatus.NOTFOUND; } - public unsafe GarnetStatus RENAME(ArgSlice oldKeySlice, ArgSlice newKeySlice, StoreType storeType) + public unsafe GarnetStatus RENAME(ArgSlice oldKeySlice, ArgSlice newKeySlice, StoreType storeType, bool withEtag) { - return RENAME(oldKeySlice, newKeySlice, storeType, false, out _); + return RENAME(oldKeySlice, newKeySlice, storeType, false, out _, withEtag); } /// @@ -562,12 +562,12 @@ public unsafe GarnetStatus RENAME(ArgSlice oldKeySlice, ArgSlice newKeySlice, St /// The new key name. /// The type of store to perform the operation on. /// - public unsafe GarnetStatus RENAMENX(ArgSlice oldKeySlice, ArgSlice newKeySlice, StoreType storeType, out int result) + public unsafe GarnetStatus RENAMENX(ArgSlice oldKeySlice, ArgSlice newKeySlice, StoreType storeType, out int result, bool withEtag) { - return RENAME(oldKeySlice, newKeySlice, storeType, true, out result); + return RENAME(oldKeySlice, newKeySlice, storeType, true, out result, withEtag); } - private unsafe GarnetStatus RENAME(ArgSlice oldKeySlice, ArgSlice newKeySlice, StoreType storeType, bool isNX, out int result) + private unsafe GarnetStatus RENAME(ArgSlice oldKeySlice, ArgSlice newKeySlice, StoreType storeType, bool isNX, out int result, bool withEtag) { RawStringInput input = default; var returnStatus = GarnetStatus.NOTFOUND; @@ -622,47 +622,64 @@ private unsafe GarnetStatus RENAME(ArgSlice oldKeySlice, ArgSlice newKeySlice, S var expirePtrVal = (byte*)expireMemoryHandle.Pointer; RespReadUtils.TryRead64Int(out var expireTimeMs, ref expirePtrVal, expirePtrVal + expireSpan.Length, out var _); - input = new RawStringInput(RespCommand.SETEXNX); + input = isNX ? new RawStringInput(RespCommand.SETEXNX) : new RawStringInput(RespCommand.SET); // If the key has an expiration, set the new key with the expiration if (expireTimeMs > 0) { - if (isNX) + if (!withEtag && !isNX) + { + SETEX(newKeySlice, newValSlice, TimeSpan.FromMilliseconds(expireTimeMs), ref context); + } + else { // Move payload forward to make space for RespInputHeader and Metadata parseState.InitializeWithArgument(newValSlice); input.parseState = parseState; input.arg1 = DateTimeOffset.UtcNow.Ticks + TimeSpan.FromMilliseconds(expireTimeMs).Ticks; + if (withEtag) + { + input.header.SetWithEtagFlag(); + } + var setStatus = SET_Conditional(ref newKey, ref input, ref context); - // For SET NX `NOTFOUND` means the operation succeeded - result = setStatus == GarnetStatus.NOTFOUND ? 1 : 0; - returnStatus = GarnetStatus.OK; - } - else - { - SETEX(newKeySlice, newValSlice, TimeSpan.FromMilliseconds(expireTimeMs), ref context); + if (isNX) + { + + // For SET NX `NOTFOUND` means the operation succeeded + result = setStatus == GarnetStatus.NOTFOUND ? 1 : 0; + returnStatus = GarnetStatus.OK; + } } } else if (expireTimeMs == -1) // Its possible to have expireTimeMs as 0 (Key expired or will be expired now) or -2 (Key does not exist), in those cases we don't SET the new key { - if (isNX) + if (!withEtag && !isNX) + { + var value = newValSlice.SpanByte; + SET(ref newKey, ref value, ref context); + } + else { // Build parse state parseState.InitializeWithArgument(newValSlice); input.parseState = parseState; + if (withEtag) + { + input.header.SetWithEtagFlag(); + } + var setStatus = SET_Conditional(ref newKey, ref input, ref context); - // For SET NX `NOTFOUND` means the operation succeeded - result = setStatus == GarnetStatus.NOTFOUND ? 1 : 0; - returnStatus = GarnetStatus.OK; - } - else - { - var value = SpanByte.FromPinnedPointer(ptrVal, headerLength); - SET(ref newKey, ref value, ref context); + if (isNX) + { + // For SET NX `NOTFOUND` means the operation succeeded + result = setStatus == GarnetStatus.NOTFOUND ? 1 : 0; + returnStatus = GarnetStatus.OK; + } } } @@ -1102,7 +1119,6 @@ public unsafe GarnetStatus Increment(ArgSlice key, out long output, lo CompletePendingForSession(ref status, ref _output, ref context); Debug.Assert(_output.IsSpanByte); - Debug.Assert(_output.Length == outputBufferLength); output = NumUtils.BytesToLong(_output.Length, outputBuffer); return GarnetStatus.OK; diff --git a/libs/server/Transaction/TransactionManager.cs b/libs/server/Transaction/TransactionManager.cs index cbdeb5cae2..1119cf84db 100644 --- a/libs/server/Transaction/TransactionManager.cs +++ b/libs/server/Transaction/TransactionManager.cs @@ -197,7 +197,7 @@ internal bool RunTransactionProc(byte id, ref CustomProcedureInput procInput, Cu // Log the transaction to AOF Log(id, ref procInput); - // Commit + // Transaction Commit Commit(); } catch (Exception ex) diff --git a/libs/server/Transaction/TxnKeyManager.cs b/libs/server/Transaction/TxnKeyManager.cs index 39b262b0a7..7238a9ecb6 100644 --- a/libs/server/Transaction/TxnKeyManager.cs +++ b/libs/server/Transaction/TxnKeyManager.cs @@ -140,7 +140,10 @@ internal int GetKeys(RespCommand command, int inputCount, out ReadOnlySpan RespCommand.HSTRLEN => HashObjectKeys((byte)HashOperation.HSTRLEN), RespCommand.HVALS => HashObjectKeys((byte)HashOperation.HVALS), RespCommand.GET => SingleKey(1, false, LockType.Shared), + RespCommand.GETIFNOTMATCH => SingleKey(1, false, LockType.Shared), + RespCommand.GETWITHETAG => SingleKey(1, false, LockType.Shared), RespCommand.SET => SingleKey(1, false, LockType.Exclusive), + RespCommand.SETIFMATCH => SingleKey(1, false, LockType.Exclusive), RespCommand.GETRANGE => SingleKey(1, false, LockType.Shared), RespCommand.SETRANGE => SingleKey(1, false, LockType.Exclusive), RespCommand.PFADD => SingleKey(1, false, LockType.Exclusive), diff --git a/libs/server/Transaction/TxnRespCommands.cs b/libs/server/Transaction/TxnRespCommands.cs index e18d97a7c7..084b004827 100644 --- a/libs/server/Transaction/TxnRespCommands.cs +++ b/libs/server/Transaction/TxnRespCommands.cs @@ -98,6 +98,8 @@ private bool NetworkEXEC() private bool NetworkSKIP(RespCommand cmd) { // Retrieve the meta-data for the command to do basic sanity checking for command arguments + // Normalize will turn internal "not-real commands" such as SETEXNX, and SETEXXX to the command info parent + cmd = cmd.NormalizeForACLs(); if (!RespCommandsInfo.TryGetRespCommandInfo(cmd, out var commandInfo, txnOnly: true, logger)) { while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_GENERIC_UNK_CMD, ref dcurr, dend)) diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordInfo.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordInfo.cs index e2176dcbc7..5d82c473f5 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordInfo.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordInfo.cs @@ -11,7 +11,7 @@ namespace Tsavorite.core { // RecordInfo layout (64 bits total): - // [Unused1][Modified][InNewVersion][Filler][Dirty][Unused2][Sealed][Valid][Tombstone][LLLLLLL] [RAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] + // [Unused1][Modified][InNewVersion][Filler][Dirty][ETag][Sealed][Valid][Tombstone][LLLLLLL] [RAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] // where L = leftover, R = readcache, A = address [StructLayout(LayoutKind.Explicit, Size = 8)] public struct RecordInfo @@ -30,8 +30,8 @@ public struct RecordInfo const int kTombstoneBitOffset = kPreviousAddressBits + kLeftoverBitCount; const int kValidBitOffset = kTombstoneBitOffset + 1; const int kSealedBitOffset = kValidBitOffset + 1; - const int kUnused2BitOffset = kSealedBitOffset + 1; - const int kDirtyBitOffset = kUnused2BitOffset + 1; + const int kEtagBitOffset = kSealedBitOffset + 1; + const int kDirtyBitOffset = kEtagBitOffset + 1; const int kFillerBitOffset = kDirtyBitOffset + 1; const int kInNewVersionBitOffset = kFillerBitOffset + 1; const int kModifiedBitOffset = kInNewVersionBitOffset + 1; @@ -40,7 +40,7 @@ public struct RecordInfo const long kTombstoneBitMask = 1L << kTombstoneBitOffset; const long kValidBitMask = 1L << kValidBitOffset; const long kSealedBitMask = 1L << kSealedBitOffset; - const long kUnused2BitMask = 1L << kUnused2BitOffset; + const long kETagBitMask = 1L << kEtagBitOffset; const long kDirtyBitMask = 1L << kDirtyBitOffset; const long kFillerBitMask = 1L << kFillerBitOffset; const long kInNewVersionBitMask = 1L << kInNewVersionBitOffset; @@ -275,18 +275,21 @@ internal bool Unused1 set => word = value ? word | kUnused1BitMask : word & ~kUnused1BitMask; } - internal bool Unused2 + public bool ETag { - readonly get => (word & kUnused2BitMask) != 0; - set => word = value ? word | kUnused2BitMask : word & ~kUnused2BitMask; + readonly get => (word & kETagBitMask) != 0; + set => word = value ? word | kETagBitMask : word & ~kETagBitMask; } + public void SetHasETag() => word |= kETagBitMask; + public void ClearHasETag() => word &= ~kETagBitMask; + public override readonly string ToString() { var paRC = IsReadCache(PreviousAddress) ? "(rc)" : string.Empty; static string bstr(bool value) => value ? "T" : "F"; return $"prev {AbsoluteAddress(PreviousAddress)}{paRC}, valid {bstr(Valid)}, tomb {bstr(Tombstone)}, seal {bstr(IsSealed)}," - + $" mod {bstr(Modified)}, dirty {bstr(Dirty)}, fill {bstr(HasFiller)}, Un1 {bstr(Unused1)}, Un2 {bstr(Unused2)}"; + + $" mod {bstr(Modified)}, dirty {bstr(Dirty)}, fill {bstr(HasFiller)}, etag {bstr(ETag)}, Un1 {bstr(Unused1)}"; } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRMW.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRMW.cs index 0652f03c90..77f01fb663 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRMW.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRMW.cs @@ -443,6 +443,9 @@ private OperationStatus CreateNewRecordRMW /// Get Span<byte> for this 's payload (excluding metadata if any) + /// + /// Optional Parameter to avoid having to call slice when wanting to interact directly with payload skipping ETag at the front of the payload + /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Span AsSpan() + public Span AsSpan(int offset = 0) { if (Serialized) - return new Span(MetadataSize + (byte*)Unsafe.AsPointer(ref payload), Length - MetadataSize); + return new Span(MetadataSize + (byte*)Unsafe.AsPointer(ref payload) + offset, Length - MetadataSize - offset); else - return new Span(MetadataSize + (byte*)payload, Length - MetadataSize); + return new Span(MetadataSize + (byte*)payload + offset, Length - MetadataSize - offset); } /// /// Get ReadOnlySpan<byte> for this 's payload (excluding metadata if any) + /// + /// Optional Parameter to avoid having to call slice when wanting to interact directly with payload skipping ETag at the front of the payload + /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ReadOnlySpan AsReadOnlySpan() + public ReadOnlySpan AsReadOnlySpan(int offset = 0) { if (Serialized) - return new ReadOnlySpan(MetadataSize + (byte*)Unsafe.AsPointer(ref payload), Length - MetadataSize); + return new ReadOnlySpan(MetadataSize + (byte*)Unsafe.AsPointer(ref payload) + offset, Length - MetadataSize - offset); else - return new ReadOnlySpan(MetadataSize + (byte*)payload, Length - MetadataSize); + return new ReadOnlySpan(MetadataSize + (byte*)payload + offset, Length - MetadataSize - offset); } /// @@ -492,6 +498,19 @@ public void CopyTo(byte* destination) } } + /// + /// Gets an Etag from the payload of the SpanByte, caller should make sure the SpanByte has an Etag for the record by checking RecordInfo + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetEtagInPayload() => *(long*)this.ToPointer(); + + /// + /// Gets an Etag from the payload of the SpanByte, caller should make sure the SpanByte has an Etag for the record by checking RecordInfo + /// + /// The Etag value to set + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void SetEtagInPayload(long etag) => *(long*)this.ToPointer() = etag; + /// public override string ToString() { diff --git a/playground/CommandInfoUpdater/SupportedCommand.cs b/playground/CommandInfoUpdater/SupportedCommand.cs index f1b5d3d67f..dbef2f2b7f 100644 --- a/playground/CommandInfoUpdater/SupportedCommand.cs +++ b/playground/CommandInfoUpdater/SupportedCommand.cs @@ -134,7 +134,9 @@ public class SupportedCommand new("GETEX", RespCommand.GETEX), new("GETBIT", RespCommand.GETBIT), new("GETDEL", RespCommand.GETDEL), + new("GETIFNOTMATCH", RespCommand.GETIFNOTMATCH), new("GETRANGE", RespCommand.GETRANGE), + new("GETWITHETAG", RespCommand.GETWITHETAG), new("GETSET", RespCommand.GETSET), new("HDEL", RespCommand.HDEL), new("HELLO", RespCommand.HELLO), @@ -236,6 +238,7 @@ public class SupportedCommand new("SET", RespCommand.SET), new("SETBIT", RespCommand.SETBIT), new("SETEX", RespCommand.SETEX), + new("SETIFMATCH", RespCommand.SETIFMATCH), new("SETNX", RespCommand.SETNX), new("SETRANGE", RespCommand.SETRANGE), new("SISMEMBER", RespCommand.SISMEMBER), diff --git a/test/Garnet.test/GarnetBitmapTests.cs b/test/Garnet.test/GarnetBitmapTests.cs index c1281a26d4..627a4984d4 100644 --- a/test/Garnet.test/GarnetBitmapTests.cs +++ b/test/Garnet.test/GarnetBitmapTests.cs @@ -165,12 +165,6 @@ public void BitmapSimpleSetGet_PCT(int bytesPerSend) public void BitmapSetGetBitTest_LTM(bool preSet) { int bitmapBytes = 512; - server.Dispose(); - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, - lowMemory: true, - MemorySize: (bitmapBytes << 2).ToString(), - PageSize: (bitmapBytes << 1).ToString()); - server.Start(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -677,6 +671,28 @@ public void BitmapBitPosTest_LTM() } } + [Test] + [Category("BITPOS")] + public void BitmapBitPosTest_BoundaryConditions() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + const int bitmapSize = 24; + byte[] bitmap = new byte[bitmapSize]; + + string key = "mybitmap"; + ClassicAssert.IsTrue(db.StringSet(key, bitmap)); + + // first unset bit, should increment + for (int i = 0; i < bitmapSize; i++) + { + // first unset bit + ClassicAssert.AreEqual(i, db.StringBitPosition(key, false)); + ClassicAssert.IsFalse(db.StringSetBit(key, i, true)); + } + } + [Test, Order(15)] [TestCase(10)] [TestCase(20)] @@ -2275,6 +2291,9 @@ public void BitmapBitPosFixedTests() pos = db.StringBitPosition(key, true, 10, 12, StringIndexType.Bit); ClassicAssert.AreEqual(10, pos); + pos = db.StringBitPosition(key, true, 20, 25, StringIndexType.Bit); + ClassicAssert.AreEqual(-1, pos); + key = "mykey2"; db.StringSetBit(key, 63, false); pos = db.StringBitPosition(key, false, 1); diff --git a/test/Garnet.test/Resp/ACL/RespCommandTests.cs b/test/Garnet.test/Resp/ACL/RespCommandTests.cs index c5341efa3a..de4ba5bdfd 100644 --- a/test/Garnet.test/Resp/ACL/RespCommandTests.cs +++ b/test/Garnet.test/Resp/ACL/RespCommandTests.cs @@ -5042,6 +5042,51 @@ static async Task DoSetKeepTtlXxAsync(GarnetClient client) } } + [Test] + public async Task SetIfMatchACLsAsync() + { + await CheckCommandsAsync( + "SETIFMATCH", + [DoSetIfMatchAsync] + ); + + static async Task DoSetIfMatchAsync(GarnetClient client) + { + var res = await client.ExecuteForStringResultAsync("SETIFMATCH", ["foo", "rizz", "0"]); + ClassicAssert.IsNull(res); + } + } + + [Test] + public async Task GetIfNotMatchACLsAsync() + { + await CheckCommandsAsync( + "GETIFNOTMATCH", + [DoGetIfNotMatchAsync] + ); + + static async Task DoGetIfNotMatchAsync(GarnetClient client) + { + var res = await client.ExecuteForStringResultAsync("GETIFNOTMATCH", ["foo", "0"]); + ClassicAssert.IsNull(res); + } + } + + [Test] + public async Task GetWithEtagACLsAsync() + { + await CheckCommandsAsync( + "GETWITHETAG", + [DoGetWithEtagAsync] + ); + + static async Task DoGetWithEtagAsync(GarnetClient client) + { + var res = await client.ExecuteForStringResultAsync("GETWITHETAG", ["foo"]); + ClassicAssert.IsNull(res); + } + } + [Test] public async Task SetBitACLsAsync() { diff --git a/test/Garnet.test/RespAofTests.cs b/test/Garnet.test/RespAofTests.cs index 9cae04773e..90e8bb61e4 100644 --- a/test/Garnet.test/RespAofTests.cs +++ b/test/Garnet.test/RespAofTests.cs @@ -229,6 +229,10 @@ public void AofRMWStoreRecoverTest() var db = redis.GetDatabase(0); db.StringSet("SeAofUpsertRecoverTestKey1", "SeAofUpsertRecoverTestValue1", expiry: TimeSpan.FromDays(1), when: When.NotExists); db.StringSet("SeAofUpsertRecoverTestKey2", "SeAofUpsertRecoverTestValue2", expiry: TimeSpan.FromDays(1), when: When.NotExists); + db.Execute("SET", "SeAofUpsertRecoverTestKey3", "SeAofUpsertRecoverTestValue3", "WITHETAG"); + db.Execute("SETIFMATCH", "SeAofUpsertRecoverTestKey3", "UpdatedSeAofUpsertRecoverTestValue3", "1"); + db.Execute("SET", "SeAofUpsertRecoverTestKey4", "2"); + var res = db.Execute("INCR", "SeAofUpsertRecoverTestKey4"); } server.Store.CommitAOF(true); @@ -243,6 +247,9 @@ public void AofRMWStoreRecoverTest() ClassicAssert.AreEqual("SeAofUpsertRecoverTestValue1", recoveredValue.ToString()); recoveredValue = db.StringGet("SeAofUpsertRecoverTestKey2"); ClassicAssert.AreEqual("SeAofUpsertRecoverTestValue2", recoveredValue.ToString()); + ExpectedEtagTest(db, "SeAofUpsertRecoverTestKey3", "UpdatedSeAofUpsertRecoverTestValue3", 2); + recoveredValue = db.StringGet("SeAofUpsertRecoverTestKey4"); + ClassicAssert.AreEqual("3", recoveredValue.ToString()); } } @@ -689,5 +696,26 @@ public void AofCustomTxnRecoverTest() ClassicAssert.AreEqual(readVal, writeKeysVal2); } } + + private static void ExpectedEtagTest(IDatabase db, string key, string expectedValue, long expected) + { + RedisResult res = db.Execute("GETWITHETAG", key); + if (expectedValue == null) + { + ClassicAssert.IsTrue(res.IsNull); + return; + } + + RedisResult[] etagAndVal = (RedisResult[])res; + RedisResult etag = etagAndVal[0]; + RedisResult val = etagAndVal[1]; + + if (expected == -1) + { + ClassicAssert.IsTrue(etag.IsNull); + } + + ClassicAssert.AreEqual(expectedValue, val.ToString()); + } } } \ No newline at end of file diff --git a/test/Garnet.test/RespCustomCommandTests.cs b/test/Garnet.test/RespCustomCommandTests.cs index 98f9af705f..7e4752c3e0 100644 --- a/test/Garnet.test/RespCustomCommandTests.cs +++ b/test/Garnet.test/RespCustomCommandTests.cs @@ -142,6 +142,86 @@ public override bool Execute(TGarnetApi garnetApi, ref CustomProcedu } } + public class CustomIncrProc : CustomProcedure + { + public override bool Execute(TGarnetApi garnetApi, ref CustomProcedureInput procInput, ref MemoryResult output) + { + var offset = 0; + var keyToIncrement = GetNextArg(ref procInput, ref offset); + + // should update etag invisibly + garnetApi.Increment(keyToIncrement, out long _, 1); + + var keyToReturn = GetNextArg(ref procInput, ref offset); + garnetApi.GET(keyToReturn, out ArgSlice outval); + WriteBulkString(ref output, outval.Span); + return true; + } + } + + public class RandomSubstituteOrExpandValForKeyTxn : CustomTransactionProcedure + { + public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) + { + int offset = 0; + AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, false); + AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, false); + return true; + } + + public override unsafe void Main(TGarnetApi garnetApi, ref CustomProcedureInput procInput, ref MemoryResult output) + { + Random rnd = new Random(); + + int offset = 0; + var key = GetNextArg(ref procInput, ref offset); + + // key will have an etag associated with it already but the transaction should not be able to see it. + // if the transaction needs to see it, then it can send GET with cmd as GETWITHETAG + garnetApi.GET(key, out ArgSlice outval); + + List valueToMessWith = outval.ToArray().ToList(); + + // random decision of either substitute, expand, or reduce value + char randChar = (char)('a' + rnd.Next(0, 26)); + int decision = rnd.Next(0, 100); + if (decision < 33) + { + // substitute + int idx = rnd.Next(0, valueToMessWith.Count); + valueToMessWith[idx] = (byte)randChar; + } + else if (decision < 66) + { + valueToMessWith.Add((byte)randChar); + } + else + { + valueToMessWith.RemoveAt(valueToMessWith.Count - 1); + } + + RawStringInput input = new RawStringInput(RespCommand.SET); + input.header.cmd = RespCommand.SET; + // if we send a SET we must explictly ask it to retain etag, and use conditional set + input.header.SetWithEtagFlag(); + + fixed (byte* valuePtr = valueToMessWith.ToArray()) + { + ArgSlice valForKey1 = new ArgSlice(valuePtr, valueToMessWith.Count); + input.parseState.InitializeWithArgument(valForKey1); + // since we are setting with retain to etag, this change should be reflected in an etag update + SpanByte sameKeyToUse = key.SpanByte; + garnetApi.SET_Conditional(ref sameKeyToUse, ref input); + } + + + var keyToIncrment = GetNextArg(ref procInput, ref offset); + + // for a non SET command the etag should be invisible and be updated automatically + garnetApi.Increment(keyToIncrment, out long _, 1); + } + } + [TestFixture] public class RespCustomCommandTests { @@ -1319,5 +1399,84 @@ public void MultiRegisterProcTest() ClassicAssert.AreEqual("65", retValue.ToString()); } } + + [Test] + public void CustomTxnEtagInteractionTest() + { + server.Register.NewTransactionProc("RANDOPS", () => new RandomSubstituteOrExpandValForKeyTxn()); + + var key1 = "key1"; + var value1 = "thisisstarting"; + + var key2 = "key2"; + var value2 = "17"; + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + try + { + db.Execute("SET", key1, value1, "WITHETAG"); + db.Execute("SET", key2, value2, "WITHETAG"); + + RedisResult result = db.Execute("RANDOPS", key1, key2); + + ClassicAssert.AreEqual("OK", result.ToString()); + + // check GETWITHETAG shows updated etag and expected values for both + RedisResult[] res = (RedisResult[])db.Execute("GETWITHETAG", key1); + ClassicAssert.AreEqual("2", res[0].ToString()); + ClassicAssert.IsTrue(res[1].ToString().All(c => c - 'a' >= 0 && c - 'a' < 26)); + + res = (RedisResult[])db.Execute("GETWITHETAG", key2); + ClassicAssert.AreEqual("2", res[0].ToString()); + ClassicAssert.AreEqual("18", res[1].ToString()); + } + catch (RedisServerException rse) + { + ClassicAssert.Fail(rse.Message); + } + } + + [Test] + public void CustomProcEtagInteractionTest() + { + server.Register.NewProcedure("INCRGET", () => new CustomIncrProc()); + + var key1 = "key1"; + var value1 = "thisisstarting"; + + var key2 = "key2"; + var value2 = "256"; + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + try + { + db.Execute("SET", key1, value1, "WITHETAG"); + db.Execute("SET", key2, value2, "WITHETAG"); + + // incr key2, and just get key1 + RedisResult result = db.Execute("INCRGET", key2, key1); + + ClassicAssert.AreEqual(value1, result.ToString()); + + // check GETWITHETAG shows updated etag and expected values for both + RedisResult[] res = (RedisResult[])db.Execute("GETWITHETAG", key1); + // etag not updated for this + ClassicAssert.AreEqual("1", res[0].ToString()); + ClassicAssert.AreEqual(value1, res[1].ToString()); + + res = (RedisResult[])db.Execute("GETWITHETAG", key2); + // etag updated for this + ClassicAssert.AreEqual("2", res[0].ToString()); + ClassicAssert.AreEqual("257", res[1].ToString()); + } + catch (RedisServerException rse) + { + ClassicAssert.Fail(rse.Message); + } + } } } \ No newline at end of file diff --git a/test/Garnet.test/RespEtagTests.cs b/test/Garnet.test/RespEtagTests.cs new file mode 100644 index 0000000000..3bd15c489e --- /dev/null +++ b/test/Garnet.test/RespEtagTests.cs @@ -0,0 +1,2095 @@ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Garnet.server; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using StackExchange.Redis; + +namespace Garnet.test +{ + [TestFixture] + public class RespEtagTests + { + private GarnetServer server; + private Random r; + + [SetUp] + public void Setup() + { + r = new Random(674386); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, disablePubSub: false); + server.Start(); + } + + [TearDown] + public void TearDown() + { + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir); + } + + #region ETAG SET Happy Paths + + [Test] + public void SETReturnsEtagForNewData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + RedisResult res = db.Execute("SET", ["rizz", "buzz", "WITHETAG"]); + long etag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(1, etag); + } + + [Test] + public void SetIfMatchReturnsNewValueAndEtagWhenEtagMatches() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + var key = "florida"; + RedisResult res = db.Execute("SET", [key, "one", "WITHETAG"]); + long initalEtag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(1, initalEtag); + + var incorrectEtag = 1738; + RedisResult[] etagMismatchMsg = (RedisResult[])db.Execute("SETIFMATCH", [key, "nextone", incorrectEtag]); + ClassicAssert.AreEqual("1", etagMismatchMsg[0].ToString()); + ClassicAssert.AreEqual("one", etagMismatchMsg[1].ToString()); + + // set a bigger val + RedisResult[] setIfMatchRes = (RedisResult[])db.Execute("SETIFMATCH", [key, "nextone", initalEtag]); + + long nextEtag = long.Parse(setIfMatchRes[0].ToString()); + var value = setIfMatchRes[1]; + + ClassicAssert.AreEqual(2, nextEtag); + ClassicAssert.IsTrue(value.IsNull); + + // set a bigger val + setIfMatchRes = (RedisResult[])db.Execute("SETIFMATCH", [key, "nextnextone", nextEtag]); + nextEtag = long.Parse(setIfMatchRes[0].ToString()); + value = setIfMatchRes[1]; + + ClassicAssert.AreEqual(3, nextEtag); + ClassicAssert.IsTrue(value.IsNull); + + // ETAGMISMATCH again + res = db.Execute("SETIFMATCH", [key, "lastOne", incorrectEtag]); + ClassicAssert.AreEqual(nextEtag.ToString(), res[0].ToString()); + ClassicAssert.AreEqual("nextnextone", res[1].ToString()); + + // set a smaller val + setIfMatchRes = (RedisResult[])db.Execute("SETIFMATCH", [key, "lastOne", nextEtag]); + nextEtag = long.Parse(setIfMatchRes[0].ToString()); + value = setIfMatchRes[1]; + + ClassicAssert.AreEqual(4, nextEtag); + ClassicAssert.IsTrue(value.IsNull); + + // ETAGMISMATCH on data that never had an etag + db.KeyDelete(key); + db.StringSet(key, "one"); + res = db.Execute("SETIFMATCH", [key, "lastOne", incorrectEtag]); + ClassicAssert.AreEqual("0", res[0].ToString()); + ClassicAssert.AreEqual("one", res[1].ToString()); + } + + [Test] + public void SetIfMatchWorksWithExpiration() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + var key = "florida"; + // Scenario: Key existed before and had no expiration + RedisResult res = db.Execute("SET", key, "one", "WITHETAG"); + long initalEtag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(1, initalEtag); + + // expiration added + long updatedEtagRes = long.Parse(db.Execute("SETIFMATCH", key, "nextone", 1, "EX", 100)[0].ToString()); + ClassicAssert.AreEqual(2, updatedEtagRes); + + // confirm expiration added -> TTL should exist + var ttl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(ttl.HasValue); + + updatedEtagRes = long.Parse(db.Execute("SETIFMATCH", key, "nextoneeexpretained", updatedEtagRes)[0].ToString()); + ClassicAssert.AreEqual(3, updatedEtagRes); + + // TTL should be retained + ttl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(ttl.HasValue); + + db.KeyDelete(key); // cleanup + + // Scenario: Key existed before and had expiration + res = db.Execute("SET", key, "one", "WITHETAG", "PX", 100000); + + // confirm expiration added -> TTL should exist + ttl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(ttl.HasValue); + + // change value and retain expiration + updatedEtagRes = long.Parse(db.Execute("SETIFMATCH", key, "nextone", 1)[0].ToString()); + ClassicAssert.AreEqual(2, updatedEtagRes); + + // TTL should be retained + ttl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(ttl.HasValue); + + // change value and change expiration + updatedEtagRes = long.Parse(db.Execute("SETIFMATCH", key, "nextoneeexpretained", 2, "EX", 100)[0].ToString()); + ClassicAssert.AreEqual(3, updatedEtagRes); + + db.KeyDelete(key); // cleanup + + // Scenario: SET without etag and existing expiration when sent with setifmatch will add etag and retain the expiration too + res = db.Execute("SET", key, "one", "EX", 100000); + // when no etag then count 0 as it's existing etag + updatedEtagRes = long.Parse(db.Execute("SETIFMATCH", key, "nextone", 0)[0].ToString()); + ClassicAssert.AreEqual(1, updatedEtagRes); + + // confirm expiration retained -> TTL should exist + ttl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(ttl.HasValue); + + // confirm has etag now + var etag = long.Parse(db.Execute("GETWITHETAG", key)[0].ToString()); + ClassicAssert.AreEqual(1, etag); + + db.KeyDelete(key); // cleanup + + + // Scenario: SET without etag and without expiration when sent with setifmatch will add etag and retain the expiration too + // copy update + res = db.Execute("SET", key, "one"); + // when no etag then count 0 as it's existing etag + updatedEtagRes = long.Parse(db.Execute("SETIFMATCH", key, "nextone", 0, "EX", 10000)[0].ToString()); + ClassicAssert.AreEqual(1, updatedEtagRes); + + // confirm expiration retained -> TTL should exist + ttl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(ttl.HasValue); + + // confirm has etag now + etag = long.Parse(db.Execute("GETWITHETAG", key)[0].ToString()); + ClassicAssert.AreEqual(1, etag); + + // same length update + res = db.Execute("SET", key, "one"); + // when no etag then count 0 as it's existing etag + updatedEtagRes = long.Parse(db.Execute("SETIFMATCH", key, "two", 0, "EX", 10000)[0].ToString()); + ClassicAssert.AreEqual(1, updatedEtagRes); + + // confirm expiration retained -> TTL should exist + ttl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(ttl.HasValue); + + // confirm has etag now + etag = long.Parse(db.Execute("GETWITHETAG", key)[0].ToString()); + ClassicAssert.AreEqual(1, etag); + + db.KeyDelete(key); // cleanup + + // Scenario: smaller length update + res = db.Execute("SET", key, "oneofusoneofus"); + // when no etag then count 0 as it's existing etag + updatedEtagRes = long.Parse(db.Execute("SETIFMATCH", key, "i", 0, "EX", 10000)[0].ToString()); + ClassicAssert.AreEqual(1, updatedEtagRes); + + // confirm expiration retained -> TTL should exist + ttl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(ttl.HasValue); + } + + #endregion + + #region ETAG GET Happy Paths + + [Test] + public void GetWithEtagReturnsValAndEtagForKey() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + var key = "florida"; + // Data that does not exist returns nil + RedisResult nonExistingData = db.Execute("GETWITHETAG", [key]); + ClassicAssert.IsTrue(nonExistingData.IsNull); + + // insert data + var initEtag = db.Execute("SET", key, "hkhalid", "WITHETAG"); + ClassicAssert.AreEqual(1, long.Parse(initEtag.ToString())); + + RedisResult[] res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + long etag = long.Parse(res[0].ToString()); + string value = res[1].ToString(); + + ClassicAssert.AreEqual(1, etag); + ClassicAssert.AreEqual("hkhalid", value); + } + + [Test] + public void GetIfNotMatchReturnsDataWhenEtagDoesNotMatch() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + var key = "florida"; + // GetIfNotMatch on non-existing data will return null + RedisResult nonExistingData = db.Execute("GETIFNOTMATCH", [key, 0]); + ClassicAssert.IsTrue(nonExistingData.IsNull); + + // insert data + var _ = db.Execute("SET", key, "maximus", "WITHETAG"); + + RedisResult[] noDataOnMatch = (RedisResult[])db.Execute("GETIFNOTMATCH", key, 1); + ClassicAssert.AreEqual("1", noDataOnMatch[0].ToString()); + ClassicAssert.IsTrue(noDataOnMatch[1].IsNull); + + RedisResult[] res = (RedisResult[])db.Execute("GETIFNOTMATCH", [key, 2]); + long etag = long.Parse(res[0].ToString()); + string value = res[1].ToString(); + + ClassicAssert.AreEqual(1, etag); + ClassicAssert.AreEqual("maximus", value); + } + + [Test] + public void SetWithEtagWorksWithMetadata() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + // Scenario: set withetag with expiration on non existing key + var key1 = "key1"; + var res1 = db.Execute("SET", key1, "value1", "WITHETAG", "EX", 10); + long etag1 = (long)res1; + ClassicAssert.AreEqual(1, etag1); + db.KeyDelete(key1); // Cleanup + + // Scenario: set with etag with expiration NX with existing key + var key2 = "key2"; + db.Execute("SET", key2, "value2", "WITHETAG"); + var res2 = db.Execute("SET", key2, "value3", "WITHETAG", "NX", "EX", 10); + ClassicAssert.IsTrue(res2.IsNull); + db.KeyDelete(key2); // Cleanup + + // Scenario: set with etag with expiration NX with non-existent key + var key3 = "key3"; + var res3 = db.Execute("SET", key3, "value4", "WITHETAG", "NX", "EX", 10); + long etag3 = (long)res3; + ClassicAssert.AreEqual(1, etag3); + db.KeyDelete(key3); // Cleanup + + // Scenario: set with etag with expiration XX + var key4 = "key4"; + db.Execute("SET", key4, "value5", "WITHETAG"); + var res4 = db.Execute("SET", key4, "value6", "WITHETAG", "XX", "EX", 10); + long etag4 = (long)res4; + ClassicAssert.AreEqual(2, etag4); + db.KeyDelete(key4); // Cleanup + + // Scenario: set with etag with expiration on existing data with etag + var key5 = "key5"; + db.Execute("SET", key5, "value7", "WITHETAG"); + var res5 = db.Execute("SET", key5, "value8", "WITHETAG", "EX", 10); + long etag5 = (long)res5; + ClassicAssert.AreEqual(2, etag5); + db.KeyDelete(key5); // Cleanup + + // Scenario: set with etag with expiration on existing data without etag + var key6 = "key6"; + db.Execute("SET", key6, "value9"); + var res6 = db.Execute("SET", key6, "value10", "WITHETAG", "EX", 10); + long etag6 = (long)res6; + ClassicAssert.AreEqual(1, etag6); + db.KeyDelete(key6); // Cleanup + + // Scenario: set with keepttl on key with etag and expiration should retain metadata and + var key7 = "key7"; + db.Execute("SET", key7, "value11", "WITHETAG", "EX", 10); + var res7 = db.Execute("SET", key7, "value12", "WITHETAG", "KEEPTTL"); + long etag7 = (long)res7; + ClassicAssert.AreEqual(2, etag7); + } + + #endregion + + # region Edgecases + + [Test] + public void SETOnAlreadyExistingSETDataOverridesItWithInitialEtag() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + RedisResult res = db.Execute("SET", "rizz", "buzz", "WITHETAG"); + long etag = (long)res; + ClassicAssert.AreEqual(1, etag); + + // update to value to update the etag + RedisResult[] updateRes = (RedisResult[])db.Execute("SETIFMATCH", ["rizz", "fixx", etag.ToString()]); + etag = (long)updateRes[0]; + ClassicAssert.AreEqual(2, etag); + ClassicAssert.IsTrue(updateRes[1].IsNull); + + // inplace update + res = db.Execute("SET", "rizz", "meow", "WITHETAG"); + etag = (long)res; + ClassicAssert.AreEqual(3, etag); + + // update to value to update the etag + updateRes = (RedisResult[])db.Execute("SETIFMATCH", ["rizz", "fooo", etag.ToString()]); + etag = (long)updateRes[0]; + ClassicAssert.AreEqual(4, etag); + ClassicAssert.IsTrue(updateRes[1].IsNull); + + // Copy update + res = db.Execute("SET", ["rizz", "oneofus", "WITHETAG"]); + etag = (long)res; + + // now we should do a getwithetag and see the etag as 0 + res = db.Execute("SET", ["rizz", "oneofus"]); + ClassicAssert.AreEqual(res.ToString(), "OK"); + + var getwithetagRes = (RedisResult[])db.Execute("GETWITHETAG", "rizz"); + ClassicAssert.AreEqual("0", getwithetagRes[0].ToString()); + } + + [Test] + public void SETWithWITHETAGOnAlreadyExistingSETDataOverridesItButUpdatesEtag() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + RedisResult res = db.Execute("SET", ["rizz", "buzz", "WITHETAG"]); + long etag = (long)res; + ClassicAssert.AreEqual(1, etag); + + // update to value to update the etag + RedisResult[] updateRes = (RedisResult[])db.Execute("SETIFMATCH", ["rizz", "fixx", etag.ToString()]); + etag = (long)updateRes[0]; + ClassicAssert.AreEqual(2, etag); + ClassicAssert.IsTrue(updateRes[1].IsNull); + + // inplace update + res = db.Execute("SET", ["rizz", "meow", "WITHETAG"]); + etag = (long)res; + ClassicAssert.AreEqual(3, etag); + + // update to value to update the etag + updateRes = (RedisResult[])db.Execute("SETIFMATCH", ["rizz", "fooo", etag.ToString()]); + etag = (long)updateRes[0]; + ClassicAssert.AreEqual(4, etag); + ClassicAssert.IsTrue(updateRes[1].IsNull); + + // Copy update + res = db.Execute("SET", ["rizz", "oneofus", "WITHETAG"]); + etag = (long)res; + ClassicAssert.AreEqual(5, etag); + } + + [Test] + public void SETWithWITHETAGOnAlreadyExistingNonEtagDataOverridesItToInitialEtag() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + ClassicAssert.IsTrue(db.StringSet("rizz", "used")); + + // inplace update + RedisResult res = db.Execute("SET", ["rizz", "buzz", "WITHETAG"]); + long etag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(1, etag); + + db.KeyDelete("rizz"); + + ClassicAssert.IsTrue(db.StringSet("rizz", "my")); + + // Copy update + res = db.Execute("SET", ["rizz", "some", "WITHETAG"]); + etag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(1, etag); + } + + #endregion + + #region ETAG Apis with non-etag data + + [Test] + public void SETOnAlreadyExistingNonEtagDataOverridesIt() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + ClassicAssert.IsTrue(db.StringSet("rizz", "used")); + + // inplace update + RedisResult res = db.Execute("SET", ["rizz", "buzz", "WITHETAG"]); + long etag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(1, etag); + + res = db.Execute("SET", ["rizz", "buzz", "WITHETAG"]); + etag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(2, etag); + + db.KeyDelete("rizz"); + + ClassicAssert.IsTrue(db.StringSet("rizz", "my")); + + // Copy update + res = db.Execute("SET", ["rizz", "some", "WITHETAG"]); + etag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(1, etag); + } + + + [Test] + public void SetIfMatchOnNonEtagDataReturnsNewEtagAndValue() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + var _ = db.StringSet("h", "k"); + + var res = (RedisResult[])db.Execute("SETIFMATCH", ["h", "t", "2"]); + ClassicAssert.AreEqual("0", res[0].ToString()); + ClassicAssert.AreEqual("k", res[1].ToString()); + } + + [Test] + public void GetIfNotMatchOnNonEtagDataReturnsNilForEtagAndCorrectData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + var _ = db.StringSet("h", "k"); + + var res = (RedisResult[])db.Execute("GETIFNOTMATCH", ["h", "1"]); + + ClassicAssert.AreEqual("0", res[0].ToString()); + ClassicAssert.AreEqual("k", res[1].ToString()); + } + + [Test] + public void GetWithEtagOnNonEtagDataReturns0ForEtagAndCorrectData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + var _ = db.StringSet("h", "k"); + + var res = (RedisResult[])db.Execute("GETWITHETAG", ["h"]); + ClassicAssert.AreEqual("0", res[0].ToString()); + ClassicAssert.AreEqual("k", res[1].ToString()); + } + + #endregion + + #region Backwards Compatability Testing + + [Test] + public void SingleEtagSetGet() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string origValue = "abcdefg"; + db.Execute("SET", ["mykey", origValue, "WITHETAG"]); + + string retValue = db.StringGet("mykey"); + + ClassicAssert.AreEqual(origValue, retValue); + } + + [Test] + public async Task SingleUnicodeEtagSetGetGarnetClient() + { + using var db = TestUtils.GetGarnetClient(); + db.Connect(); + + string origValue = "笑い男"; + await db.ExecuteForLongResultAsync("SET", ["mykey", origValue, "WITHETAG"]); + + string retValue = await db.StringGetAsync("mykey"); + + ClassicAssert.AreEqual(origValue, retValue); + } + + [Test] + public async Task LargeEtagSetGet() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + const int length = (1 << 19) + 100; + var value = new byte[length]; + + for (int i = 0; i < length; i++) + value[i] = (byte)((byte)'a' + ((byte)i % 26)); + + RedisResult res = await db.ExecuteAsync("SET", ["mykey", value, "WITHETAG"]); + long initalEtag = long.Parse(res.ToString()); + ClassicAssert.AreEqual(1, initalEtag); + + // Backwards compatability of data set with etag and plain GET call + var retvalue = (byte[])await db.StringGetAsync("mykey"); + + ClassicAssert.IsTrue(new ReadOnlySpan(value).SequenceEqual(new ReadOnlySpan(retvalue))); + } + + [Test] + public void SetExpiryForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string origValue = "abcdefghij"; + + // set with etag + long initalEtag = long.Parse(db.Execute("SET", ["mykey", origValue, "EX", 2, "WITHETAG"]).ToString()); + ClassicAssert.AreEqual(1, initalEtag); + + string retValue = db.StringGet("mykey"); + ClassicAssert.AreEqual(origValue, retValue, "Get() before expiration"); + + var actualDbSize = db.Execute("DBSIZE"); + ClassicAssert.AreEqual(1, (ulong)actualDbSize, "DBSIZE before expiration"); + + var actualKeys = db.Execute("KEYS", ["*"]); + ClassicAssert.AreEqual(1, ((RedisResult[])actualKeys).Length, "KEYS before expiration"); + + var actualScan = db.Execute("SCAN", "0"); + ClassicAssert.AreEqual(1, ((RedisValue[])((RedisResult[])actualScan!)[1]).Length, "SCAN before expiration"); + + Thread.Sleep(2500); + + retValue = db.StringGet("mykey"); + ClassicAssert.AreEqual(null, retValue, "Get() after expiration"); + + actualDbSize = db.Execute("DBSIZE"); + ClassicAssert.AreEqual(0, (ulong)actualDbSize, "DBSIZE after expiration"); + + actualKeys = db.Execute("KEYS", ["*"]); + ClassicAssert.AreEqual(0, ((RedisResult[])actualKeys).Length, "KEYS after expiration"); + + actualScan = db.Execute("SCAN", "0"); + ClassicAssert.AreEqual(0, ((RedisValue[])((RedisResult[])actualScan!)[1]).Length, "SCAN after expiration"); + } + + [Test] + public void SetExpiryHighPrecisionForEtagSetDatat() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var origValue = "abcdeghijklmno"; + // set with etag + long initalEtag = long.Parse(db.Execute("SET", ["mykey", origValue, "PX", 1900, "WITHETAG"]).ToString()); + ClassicAssert.AreEqual(1, initalEtag); + + string retValue = db.StringGet("mykey"); + ClassicAssert.AreEqual(origValue, retValue); + + Thread.Sleep(1000); + retValue = db.StringGet("mykey"); + ClassicAssert.AreEqual(origValue, retValue); + + Thread.Sleep(2000); + retValue = db.StringGet("mykey"); + ClassicAssert.AreEqual(null, retValue); + } + + [Test] + public void SetExpiryIncrForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Key storing integer + var nVal = -100000; + var strKey = "key1"; + db.Execute("SET", [strKey, nVal, "WITHETAG"]); + db.KeyExpire(strKey, TimeSpan.FromSeconds(5)); + + string res1 = db.StringGet(strKey); + + long n = db.StringIncrement(strKey); + + // This should increase the ETAG internally so we have a check for that here + var checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(2, checkEtag); + + string res = db.StringGet(strKey); + long nRetVal = Convert.ToInt64(res); + ClassicAssert.AreEqual(n, nRetVal); + ClassicAssert.AreEqual(-99999, nRetVal); + + n = db.StringIncrement(strKey); + + // This should increase the ETAG internally so we have a check for that here + checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(3, checkEtag); + + nRetVal = Convert.ToInt64(db.StringGet(strKey)); + ClassicAssert.AreEqual(n, nRetVal); + ClassicAssert.AreEqual(-99998, nRetVal); + + var res69 = db.KeyTimeToLive(strKey); + + Thread.Sleep(5000); + + // Expired key, restart increment,after exp this is treated as new record + n = db.StringIncrement(strKey); + ClassicAssert.AreEqual(1, n); + + nRetVal = Convert.ToInt64(db.StringGet(strKey)); + ClassicAssert.AreEqual(1, nRetVal); + + var etagGet = (RedisResult[])db.Execute("GETWITHETAG", [strKey]); + // Etag will show up as 0 since the previous one had expired + ClassicAssert.AreEqual("0", etagGet[0].ToString()); + ClassicAssert.AreEqual(1, Convert.ToInt64(etagGet[1])); + } + + [Test] + public void IncrDecrChangeDigitsWithExpiry() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var strKey = "key1"; + + db.Execute("SET", [strKey, 9, "WITHETAG"]); + + long checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(1, checkEtag); + + db.KeyExpire(strKey, TimeSpan.FromSeconds(5)); + + long n = db.StringIncrement(strKey); + long nRetVal = Convert.ToInt64(db.StringGet(strKey)); + ClassicAssert.AreEqual(n, nRetVal); + ClassicAssert.AreEqual(10, nRetVal); + + checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(2, checkEtag); + + n = db.StringDecrement(strKey); + nRetVal = Convert.ToInt64(db.StringGet(strKey)); + ClassicAssert.AreEqual(n, nRetVal); + ClassicAssert.AreEqual(9, nRetVal); + + checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(3, checkEtag); + + Thread.Sleep(TimeSpan.FromSeconds(5)); + + var res = (string)db.StringGet(strKey); + ClassicAssert.IsNull(res); + } + + [Test] + public void StringSetOnAnExistingEtagDataOverrides() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var strKey = "mykey"; + db.Execute("SET", [strKey, 9, "WITHETAG"]); + + long checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(1, checkEtag); + + // Unless the SET was called with WITHETAG a call to set will override the SET to a new + // value altogether, this will make it lose it's etag capability. This is a limitation for Etags + // because plain sets are upserts (blind updates), and currently we cannot increase the latency in + // the common path for set to check beyong Readonly address for the existence of a record with ETag. + // This means that sets are complete upserts and clients need to use setifmatch, or set with WITHETAG + // if they want each consequent set to maintain the key value pair's etag property. + ClassicAssert.IsTrue(db.StringSet(strKey, "ciaociao")); + + string retVal = db.StringGet(strKey).ToString(); + ClassicAssert.AreEqual("ciaociao", retVal); + + var res = (RedisResult[])db.Execute("GETWITHETAG", [strKey]); + ClassicAssert.AreEqual("0", res[0].ToString()); + ClassicAssert.AreEqual("ciaociao", res[1].ToString()); + } + + [Test] + public void StringSetOnAnExistingEtagDataUpdatesEtagIfEtagRetain() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var strKey = "mykey"; + db.Execute("SET", strKey, "9", "WITHETAG"); + + long checkEtag = (long)db.Execute("GETWITHETAG", [strKey])[0]; + ClassicAssert.AreEqual(1, checkEtag); + + // Unless you explicitly call SET with WITHETAG option you will lose the etag on the previous key-value pair + db.Execute("SET", [strKey, "ciaociao", "WITHETAG"]); + + string retVal = db.StringGet(strKey).ToString(); + ClassicAssert.AreEqual("ciaociao", retVal); + + var res = (RedisResult[])db.Execute("GETWITHETAG", strKey); + ClassicAssert.AreEqual(2, (long)res[0]); + + // on subsequent upserts we are still increasing the etag transparently + db.Execute("SET", [strKey, "ciaociaociao", "WITHETAG"]); + + retVal = db.StringGet(strKey).ToString(); + ClassicAssert.AreEqual("ciaociaociao", retVal); + + res = (RedisResult[])db.Execute("GETWITHETAG", strKey); + ClassicAssert.AreEqual(3, (long)res[0]); + ClassicAssert.AreEqual("ciaociaociao", res[1].ToString()); + } + + [Test] + public void LockTakeReleaseOnAValueInitiallySET() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string key = "lock-key"; + string value = "lock-value"; + + var initalEtag = long.Parse(db.Execute("SET", [key, value, "WITHETAG"]).ToString()); + ClassicAssert.AreEqual(1, initalEtag); + + var success = db.LockTake(key, value, TimeSpan.FromSeconds(100)); + ClassicAssert.IsFalse(success); + + success = db.LockRelease(key, value); + ClassicAssert.IsTrue(success); + + success = db.LockRelease(key, value); + ClassicAssert.IsFalse(success); + + success = db.LockTake(key, value, TimeSpan.FromSeconds(100)); + ClassicAssert.IsTrue(success); + + success = db.LockRelease(key, value); + ClassicAssert.IsTrue(success); + + // Test auto-lock-release + success = db.LockTake(key, value, TimeSpan.FromSeconds(1)); + ClassicAssert.IsTrue(success); + + Thread.Sleep(2000); + success = db.LockTake(key, value, TimeSpan.FromSeconds(1)); + ClassicAssert.IsTrue(success); + + success = db.LockRelease(key, value); + ClassicAssert.IsTrue(success); + } + + [Test] + [TestCase("key1", 1000)] + [TestCase("key1", 0)] + public void SingleDecrForEtagSetData(string strKey, int nVal) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Key storing integer + var initalEtag = long.Parse(db.Execute("SET", [strKey, nVal, "WITHETAG"]).ToString()); + ClassicAssert.AreEqual(1, initalEtag); + + long n = db.StringDecrement(strKey); + ClassicAssert.AreEqual(nVal - 1, n); + long nRetVal = Convert.ToInt64(db.StringGet(strKey)); + ClassicAssert.AreEqual(n, nRetVal); + + long checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(2, checkEtag); + } + + [Test] + [TestCase(-1000, 100)] + [TestCase(-1000, -9000)] + [TestCase(-10000, 9000)] + [TestCase(9000, 10000)] + public void SingleDecrByForEtagSetData(long nVal, long nDecr) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + // Key storing integer val + var strKey = "key1"; + var initalEtag = long.Parse(db.Execute("SET", [strKey, nVal, "WITHETAG"]).ToString()); + ClassicAssert.AreEqual(1, initalEtag); + + long n = db.StringDecrement(strKey, nDecr); + + int nRetVal = Convert.ToInt32(db.StringGet(strKey)); + ClassicAssert.AreEqual(n, nRetVal); + + long checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); + ClassicAssert.AreEqual(2, checkEtag); + } + + [Test] + [TestCase(RespCommand.INCR)] + [TestCase(RespCommand.DECR)] + [TestCase(RespCommand.INCRBY)] + [TestCase(RespCommand.DECRBY)] + public void SimpleIncrementInvalidValueForEtagSetdata(RespCommand cmd) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + string[] values = ["", "7 3", "02+(34", "笑い男", "01", "-01", "7ab"]; + + for (var i = 0; i < values.Length; i++) + { + var key = $"key{i}"; + var exception = false; + var initalEtag = long.Parse(db.Execute("SET", [key, values[i], "WITHETAG"]).ToString()); + ClassicAssert.AreEqual(1, initalEtag); + + try + { + _ = cmd switch + { + RespCommand.INCR => db.StringIncrement(key), + RespCommand.DECR => db.StringDecrement(key), + RespCommand.INCRBY => db.StringIncrement(key, 10L), + RespCommand.DECRBY => db.StringDecrement(key, 10L), + _ => throw new Exception($"Command {cmd} not supported!"), + }; + } + catch (Exception ex) + { + exception = true; + var msg = ex.Message; + ClassicAssert.AreEqual("ERR value is not an integer or out of range.", msg); + } + ClassicAssert.IsTrue(exception); + } + } + + [Test] + [TestCase(RespCommand.INCR)] + [TestCase(RespCommand.DECR)] + [TestCase(RespCommand.INCRBY)] + [TestCase(RespCommand.DECRBY)] + public void SimpleIncrementOverflowForEtagSetData(RespCommand cmd) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var exception = false; + + var key = "test"; + + try + { + switch (cmd) + { + case RespCommand.INCR: + _ = db.Execute("SET", [key, long.MaxValue.ToString(), "WITHETAG"]); + _ = db.StringIncrement(key); + break; + case RespCommand.DECR: + _ = db.Execute("SET", [key, long.MinValue.ToString(), "WITHETAG"]); + _ = db.StringDecrement(key); + break; + case RespCommand.INCRBY: + _ = db.Execute("SET", [key, 0, "WITHETAG"]); + _ = db.Execute("INCRBY", [key, ulong.MaxValue.ToString()]); + break; + case RespCommand.DECRBY: + _ = db.Execute("SET", [key, 0, "WITHETAG"]); + _ = db.Execute("DECRBY", [key, ulong.MaxValue.ToString()]); + break; + } + } + catch (Exception ex) + { + exception = true; + var msg = ex.Message; + ClassicAssert.AreEqual("ERR value is not an integer or out of range.", msg); + } + ClassicAssert.IsTrue(exception); + } + + [Test] + [TestCase(0, 12.6)] + [TestCase(12.6, 0)] + [TestCase(10, 10)] + [TestCase(910151, 0.23659)] + [TestCase(663.12336412, 12342.3)] + [TestCase(10, -110)] + [TestCase(110, -110.234)] + [TestCase(-2110.95255555, -110.234)] + [TestCase(-2110.95255555, 100000.526654512219412)] + [TestCase(double.MaxValue, double.MinValue)] + public void SimpleIncrementByFloatForEtagSetData(double initialValue, double incrByValue) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key1"; + db.Execute("SET", key, initialValue, "WITHETAG"); + + var expectedResult = initialValue + incrByValue; + + var actualResultStr = (string)db.Execute("INCRBYFLOAT", [key, incrByValue]); + var actualResultRawStr = db.StringGet(key); + + var actualResult = double.Parse(actualResultStr, CultureInfo.InvariantCulture); + var actualResultRaw = double.Parse(actualResultRawStr, CultureInfo.InvariantCulture); + + Assert.That(actualResult, Is.EqualTo(expectedResult).Within(1.0 / Math.Pow(10, 15))); + Assert.That(actualResult, Is.EqualTo(actualResultRaw).Within(1.0 / Math.Pow(10, 15))); + + RedisResult[] res = (RedisResult[])db.Execute("GETWITHETAG", key); + long etag = (long)res[0]; + double value = double.Parse(res[1].ToString(), CultureInfo.InvariantCulture); + Assert.That(value, Is.EqualTo(actualResultRaw).Within(1.0 / Math.Pow(10, 15))); + ClassicAssert.AreEqual(2, etag); + } + + [Test] + public void SingleDeleteForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Key storing integer + var nVal = 100; + var strKey = "key1"; + db.Execute("SET", [strKey, nVal, "WITHETAG"]); + db.KeyDelete(strKey); + var retVal = Convert.ToBoolean(db.StringGet(strKey)); + ClassicAssert.AreEqual(retVal, false); + } + + [Test] + public void SingleDeleteWithObjectStoreDisabledForEtagSetData() + { + TearDown(); + + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); + server.Start(); + + var key = "delKey"; + var value = "1234"; + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("SET", [key, value, "WITHETAG"]); + + var resp = (string)db.StringGet(key); + ClassicAssert.AreEqual(resp, value); + + var respDel = db.KeyDelete(key); + ClassicAssert.IsTrue(respDel); + + respDel = db.KeyDelete(key); + ClassicAssert.IsFalse(respDel); + } + + private string GetRandomString(int len) + { + const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + return new string(Enumerable.Repeat(chars, len) + .Select(s => s[r.Next(s.Length)]).ToArray()); + } + + [Test] + public void SingleDeleteWithObjectStoreDisable_LTMForEtagSetData() + { + TearDown(); + + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, DisableObjects: true); + server.Start(); + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + int keyCount = 5; + int valLen = 256; + int keyLen = 8; + + List> data = []; + for (int i = 0; i < keyCount; i++) + { + data.Add(new Tuple(GetRandomString(keyLen), GetRandomString(valLen))); + var pair = data.Last(); + db.Execute("SET", [pair.Item1, pair.Item2, "WITHETAG"]); + } + + + for (int i = 0; i < keyCount; i++) + { + var pair = data[i]; + + var resp = (string)db.StringGet(pair.Item1); + ClassicAssert.AreEqual(resp, pair.Item2); + + var respDel = db.KeyDelete(pair.Item1); + resp = (string)db.StringGet(pair.Item1); + ClassicAssert.IsNull(resp); + + respDel = db.KeyDelete(pair.Item2); + ClassicAssert.IsFalse(respDel); + } + } + + [Test] + public void MultiKeyDeleteForEtagSetData([Values] bool withoutObjectStore) + { + if (withoutObjectStore) + { + TearDown(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); + server.Start(); + } + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + int keyCount = 10; + int valLen = 16; + int keyLen = 8; + + List> data = []; + for (int i = 0; i < keyCount; i++) + { + data.Add(new Tuple(GetRandomString(keyLen), GetRandomString(valLen))); + var pair = data.Last(); + db.Execute("SET", [pair.Item1, pair.Item2, "WITHETAG"]); + } + + var keys = data.Select(x => (RedisKey)x.Item1).ToArray(); + var keysDeleted = db.KeyDeleteAsync(keys); + keysDeleted.Wait(); + ClassicAssert.AreEqual(keysDeleted.Result, 10); + + var keysDel = db.KeyDelete(keys); + ClassicAssert.AreEqual(keysDel, 0); + } + + [Test] + public void MultiKeyUnlinkForEtagSetData([Values] bool withoutObjectStore) + { + if (withoutObjectStore) + { + TearDown(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); + server.Start(); + } + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + int keyCount = 10; + int valLen = 16; + int keyLen = 8; + + List> data = []; + for (int i = 0; i < keyCount; i++) + { + data.Add(new Tuple(GetRandomString(keyLen), GetRandomString(valLen))); + var pair = data.Last(); + db.Execute("SET", [pair.Item1, pair.Item2, "WITHETAG"]); + } + + var keys = data.Select(x => (object)x.Item1).ToArray(); + var keysDeleted = (string)db.Execute("unlink", keys); + ClassicAssert.AreEqual(10, int.Parse(keysDeleted)); + + keysDeleted = (string)db.Execute("unlink", keys); + ClassicAssert.AreEqual(0, int.Parse(keysDeleted)); + } + + [Test] + public void SingleExistsForEtagSetData([Values] bool withoutObjectStore) + { + if (withoutObjectStore) + { + TearDown(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, DisableObjects: true); + server.Start(); + } + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Key storing integer + var nVal = 100; + var strKey = "key1"; + ClassicAssert.IsFalse(db.KeyExists(strKey)); + + db.Execute("SET", [strKey, nVal, "WITHETAG"]); + + bool fExists = db.KeyExists("key1", CommandFlags.None); + ClassicAssert.AreEqual(fExists, true); + + fExists = db.KeyExists("key2", CommandFlags.None); + ClassicAssert.AreEqual(fExists, false); + } + + + [Test] + public void MultipleExistsKeysAndObjectsAndEtagData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var count = db.ListLeftPush("listKey", ["a", "b", "c", "d"]); + ClassicAssert.AreEqual(4, count); + + var zaddItems = db.SortedSetAdd("zset:test", [new SortedSetEntry("a", 1), new SortedSetEntry("b", 2)]); + ClassicAssert.AreEqual(2, zaddItems); + + db.StringSet("foo", "bar"); + + db.Execute("SET", ["rizz", "bar", "WITHETAG"]); + + var exists = db.KeyExists(["key", "listKey", "zset:test", "foo", "rizz"]); + ClassicAssert.AreEqual(4, exists); + } + + #region RENAME + + + [Test] + public void RenameEtagTests() + { + // old key had etag => new key zero'd etag when made without withetag (new key did not exists) + // old key had etag => new key zero'd etag when made without withetag (new key exists without etag) + // old key had etag => new key has updated etag when made with withetag (new key exists withetag) + // old key not have etag => new key made with updated etag when made withetag (new key did exist withetag) + // old key had etag and, new key has initial etag when made with withetag (new key did not exists) + // old key not have etag and, new key made with initial etag when made withetag (new key did not exist) + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + string origValue = "test1"; + string oldKey = "key1"; + string newKey = "key2"; + + // Scenario: old key had etag and => new key zero'd etag when made without withetag (new key did not exists) + + long etag = long.Parse(db.Execute("SET", [oldKey, origValue, "WITHETAG"]).ToString()); + ClassicAssert.AreEqual(1, etag); + + db.KeyRename(oldKey, newKey); + + ClassicAssert.IsTrue(db.StringGet(oldKey).IsNull); + ClassicAssert.IsTrue(EtagAndValMatches(db, newKey, 0, origValue)); + // old key has been deleted, and new key exists without etag at this point + + // Scenario: old key had etag => new key zero'd etag when made without withetag (new key exists without etag) + db.Execute("SET", oldKey, origValue, "WITHETAG"); + + db.KeyRename(oldKey, newKey); + + ClassicAssert.IsTrue(db.StringGet(oldKey).IsNull); + ClassicAssert.IsTrue(EtagAndValMatches(db, newKey, 0, origValue)); + db.KeyDelete(newKey); + + // Scenario: old key had etag => new key has updated etag when made with withetag (new key exists withetag) + // setup new key with updated etag + db.Execute("SET", newKey, origValue + "delta", "WITHETAG"); + db.Execute("SETIFMATCH", newKey, origValue, 1); // updates etag to 2 + // old key with etag + etag = long.Parse(db.Execute("SET", [oldKey, origValue, "WITHETAG"]).ToString()); + ClassicAssert.AreEqual(1, etag); + + db.Execute("RENAME", oldKey, newKey, "WITHETAG"); // should update etag to 3 + + ClassicAssert.IsTrue(db.StringGet(oldKey).IsNull); + ClassicAssert.IsTrue(EtagAndValMatches(db, newKey, 3, origValue)); + // at this point new key exists with etag, old key does not exist at all + + // Scenario: old key not have etag => new key made with updated etag when made withetag (new key did exist withetag) + db.Execute("SET", oldKey, origValue); + + db.Execute("RENAME", oldKey, newKey, "WITHETAG"); + + ClassicAssert.IsTrue(db.StringGet(oldKey).IsNull); + ClassicAssert.IsTrue(EtagAndValMatches(db, newKey, 4, origValue)); + db.KeyDelete(newKey); + + // Scenario: old key had etag => new key has initial etag when made with withetag (new key did not exists) + db.Execute("SET", oldKey, origValue, "WITHETAG"); + + db.Execute("RENAME", oldKey, newKey, "WITHETAG"); + + ClassicAssert.IsTrue(db.StringGet(oldKey).IsNull); + ClassicAssert.IsTrue(EtagAndValMatches(db, newKey, 1, origValue)); + db.KeyDelete(newKey); + + // Scenario: old key not have etag => new key made with initial etag when made withetag (new key did not exist) + db.Execute("SET", oldKey, origValue); + + db.Execute("RENAME", oldKey, newKey, "WITHETAG"); + + ClassicAssert.IsTrue(db.StringGet(oldKey).IsNull); + ClassicAssert.IsTrue(EtagAndValMatches(db, newKey, 1, origValue)); + db.KeyDelete(newKey); + } + + private bool EtagAndValMatches(IDatabase db, string key, long expectedEtag, string expectedValue) + { + var res = (RedisResult[])db.Execute("GETWITHETAG", key); + var responseEtag = long.Parse(res[0].ToString()); + var responseValue = res[1].ToString(); + return responseValue == expectedValue && responseEtag == expectedEtag; + } + + #endregion + + [Test] + public void PersistTTLTestForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "expireKey"; + var val = "expireValue"; + var expire = 2; + + var ttl = db.Execute("TTL", key); + ClassicAssert.AreEqual(-2, (int)ttl); + + db.Execute("SET", [key, val, "WITHETAG"]); + ttl = db.Execute("TTL", key); + ClassicAssert.AreEqual(-1, (int)ttl); + + db.KeyExpire(key, TimeSpan.FromSeconds(expire)); + + var res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + ClassicAssert.AreEqual(1, long.Parse(res[0].ToString())); + ClassicAssert.AreEqual(val, res[1].ToString()); + + var time = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(time.Value.TotalSeconds > 0); + + db.KeyExpire(key, TimeSpan.FromSeconds(expire)); + res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + ClassicAssert.AreEqual(1, long.Parse(res[0].ToString())); + ClassicAssert.AreEqual(val, res[1].ToString()); + + db.KeyPersist(key); + res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + // unchanged etag + ClassicAssert.AreEqual(1, long.Parse(res[0].ToString())); + ClassicAssert.AreEqual(val, res[1].ToString()); + + Thread.Sleep((expire + 1) * 1000); + + var _val = db.StringGet(key); + ClassicAssert.AreEqual(val, _val.ToString()); + + time = db.KeyTimeToLive(key); + ClassicAssert.IsNull(time); + + res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + // the tag was persisted along with data from persist despite previous TTL + ClassicAssert.AreEqual(1, long.Parse(res[0].ToString())); + ClassicAssert.AreEqual(val, res[1].ToString()); + } + + [Test] + public void PersistTestForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + int expire = 100; + var keyA = "keyA"; + db.Execute("SET", [keyA, keyA, "WITHETAG"]); + + var response = db.KeyPersist(keyA); + ClassicAssert.IsFalse(response); + + db.KeyExpire(keyA, TimeSpan.FromSeconds(expire)); + var time = db.KeyTimeToLive(keyA); + ClassicAssert.IsTrue(time.Value.TotalSeconds > 0); + + response = db.KeyPersist(keyA); + ClassicAssert.IsTrue(response); + + time = db.KeyTimeToLive(keyA); + ClassicAssert.IsTrue(time == null); + + var value = db.StringGet(keyA); + ClassicAssert.AreEqual(value, keyA); + + var res = (RedisResult[])db.Execute("GETWITHETAG", [keyA]); + ClassicAssert.AreEqual(1, long.Parse(res[0].ToString())); + ClassicAssert.AreEqual(keyA, res[1].ToString()); + + var noKey = "noKey"; + response = db.KeyPersist(noKey); + ClassicAssert.IsFalse(response); + } + + [Test] + [TestCase("EXPIRE")] + [TestCase("PEXPIRE")] + public void KeyExpireStringTestForEtagSetData(string command) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "keyA"; + db.Execute("SET", [key, key, "WITHETAG"]); + + var value = db.StringGet(key); + ClassicAssert.AreEqual(key, (string)value); + + if (command.Equals("EXPIRE")) + db.KeyExpire(key, TimeSpan.FromSeconds(1)); + else + db.Execute(command, [key, 1000]); + + Thread.Sleep(1500); + + value = db.StringGet(key); + ClassicAssert.AreEqual(null, (string)value); + } + + [Test] + [TestCase("EXPIRE")] + [TestCase("PEXPIRE")] + public void KeyExpireOptionsTestForEtagSetData(string command) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "keyA"; + object[] args = [key, 1000, ""]; + db.Execute("SET", [key, key, "WITHETAG"]); + + args[2] = "XX";// XX -- Set expiry only when the key has an existing expiry + bool resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsFalse(resp);//XX return false no existing expiry + + args[2] = "NX";// NX -- Set expiry only when the key has no expiry + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsTrue(resp);// NX return true no existing expiry + + args[2] = "NX";// NX -- Set expiry only when the key has no expiry + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsFalse(resp);// NX return false existing expiry + + args[1] = 50; + args[2] = "XX";// XX -- Set expiry only when the key has an existing expiry + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsTrue(resp);// XX return true existing expiry + var time = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(time.Value.TotalSeconds <= (double)((int)args[1]) && time.Value.TotalSeconds > 0); + + args[1] = 1; + args[2] = "GT";// GT -- Set expiry only when the new expiry is greater than current one + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsFalse(resp); // GT return false new expiry < current expiry + + args[1] = 1000; + args[2] = "GT";// GT -- Set expiry only when the new expiry is greater than current one + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsTrue(resp); // GT return true new expiry > current expiry + time = db.KeyTimeToLive(key); + + if (command.Equals("EXPIRE")) + ClassicAssert.IsTrue(time.Value.TotalSeconds > 500); + else + ClassicAssert.IsTrue(time.Value.TotalMilliseconds > 500); + + args[1] = 2000; + args[2] = "LT";// LT -- Set expiry only when the new expiry is less than current one + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsFalse(resp); // LT return false new expiry > current expiry + + args[1] = 15; + args[2] = "LT";// LT -- Set expiry only when the new expiry is less than current one + resp = (bool)db.Execute($"{command}", args); + ClassicAssert.IsTrue(resp); // LT return true new expiry < current expiry + time = db.KeyTimeToLive(key); + + if (command.Equals("EXPIRE")) + ClassicAssert.IsTrue(time.Value.TotalSeconds <= (double)((int)args[1]) && time.Value.TotalSeconds > 0); + else + ClassicAssert.IsTrue(time.Value.TotalMilliseconds <= (double)((int)args[1]) && time.Value.TotalMilliseconds > 0); + } + + [Test] + public void MainObjectKeyForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var server = redis.GetServers()[0]; + var db = redis.GetDatabase(0); + + const string key = "test:1"; + + ClassicAssert.AreEqual(1, long.Parse(db.Execute("SET", key, "v1", "WITHETAG").ToString())); + + // Do SetAdd using the same key + ClassicAssert.IsTrue(db.SetAdd(key, "v2")); + + // Two keys "test:1" - this is expected as of now + // because Garnet has a separate main and object store + var keys = server.Keys(db.Database, key).ToList(); + ClassicAssert.AreEqual(2, keys.Count); + ClassicAssert.AreEqual(key, (string)keys[0]); + ClassicAssert.AreEqual(key, (string)keys[1]); + + // do ListRightPush using the same key, expected error + var ex = Assert.Throws(() => db.ListRightPush(key, "v3")); + var expectedError = Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE); + ClassicAssert.IsNotNull(ex); + ClassicAssert.AreEqual(expectedError, ex.Message); + } + + [Test] + public void GetSliceTestForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string key = "rangeKey"; + string value = "0123456789"; + + var resp = (string)db.StringGetRange(key, 2, 10); + ClassicAssert.AreEqual(string.Empty, resp); + + ClassicAssert.AreEqual(1, long.Parse(db.Execute("SET", key, value, "WITHETAG").ToString())); + + //0,0 + resp = (string)db.StringGetRange(key, 0, 0); + ClassicAssert.AreEqual("0", resp); + + //actual value + resp = (string)db.StringGetRange(key, 0, -1); + ClassicAssert.AreEqual(value, resp); + + #region testA + //s[2,len] s < e & e = len + resp = (string)db.StringGetRange(key, 2, 10); + ClassicAssert.AreEqual(value.Substring(2), resp); + + //s[2,len] s < e & e = len - 1 + resp = (string)db.StringGetRange(key, 2, 9); + ClassicAssert.AreEqual(value.Substring(2), resp); + + //s[2,len] s < e < len + resp = (string)db.StringGetRange(key, 2, 5); + ClassicAssert.AreEqual(value.Substring(2, 4), resp); + + //s[2,len] s < len < e + resp = (string)db.StringGetRange(key, 2, 15); + ClassicAssert.AreEqual(value.Substring(2), resp); + + //s[4,len] e < s < len + resp = (string)db.StringGetRange(key, 4, 2); + ClassicAssert.AreEqual("", resp); + + //s[4,len] e < 0 < s < len + resp = (string)db.StringGetRange(key, 4, -2); + ClassicAssert.AreEqual(value.Substring(4, 5), resp); + + //s[4,len] e < -len < 0 < s < len + resp = (string)db.StringGetRange(key, 4, -12); + ClassicAssert.AreEqual("", resp); + #endregion + + #region testB + //-len < s < 0 < len < e + resp = (string)db.StringGetRange(key, -4, 15); + ClassicAssert.AreEqual(value.Substring(6, 4), resp); + + //-len < s < 0 < e < len where len + s > e + resp = (string)db.StringGetRange(key, -4, 5); + ClassicAssert.AreEqual("", resp); + + //-len < s < 0 < e < len where len + s < e + resp = (string)db.StringGetRange(key, -4, 8); + ClassicAssert.AreEqual(value.Substring(value.Length - 4, 2), resp); + + //-len < s < e < 0 + resp = (string)db.StringGetRange(key, -4, -1); + ClassicAssert.AreEqual(value.Substring(value.Length - 4, 4), resp); + + //-len < e < s < 0 + resp = (string)db.StringGetRange(key, -4, -7); + ClassicAssert.AreEqual("", resp); + #endregion + + //range start > end > len + resp = (string)db.StringGetRange(key, 17, 13); + ClassicAssert.AreEqual("", resp); + + //range 0 > start > end + resp = (string)db.StringGetRange(key, -1, -4); + ClassicAssert.AreEqual("", resp); + + //equal offsets + resp = db.StringGetRange(key, 4, 4); + ClassicAssert.AreEqual("4", resp); + + //equal offsets + resp = db.StringGetRange(key, -4, -4); + ClassicAssert.AreEqual("6", resp); + + //equal offsets + resp = db.StringGetRange(key, -100, -100); + ClassicAssert.AreEqual("0", resp); + + //equal offsets + resp = db.StringGetRange(key, -101, -101); + ClassicAssert.AreEqual("9", resp); + + //start larger than end + resp = db.StringGetRange(key, -1, -3); + ClassicAssert.AreEqual("", resp); + + //2,-1 -> 2 9 + var negend = -1; + resp = db.StringGetRange(key, 2, negend); + ClassicAssert.AreEqual(value.Substring(2, 8), resp); + + //2,-3 -> 2 7 + negend = -3; + resp = db.StringGetRange(key, 2, negend); + ClassicAssert.AreEqual(value.Substring(2, 6), resp); + + //-5,-3 -> 5,7 + var negstart = -5; + resp = db.StringGetRange(key, negstart, negend); + ClassicAssert.AreEqual(value.Substring(5, 3), resp); + } + + [Test] + public void SetRangeTestForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string key = "setRangeKey"; + string value = "0123456789"; + string newValue = "ABCDE"; + + db.Execute("SET", key, value, "WITHETAG"); + + var resp = db.StringGet(key); + ClassicAssert.AreEqual("0123456789", resp.ToString()); + + // new key, length 10, offset 5 -> 15 ("\0\0\0\0\00123456789") + resp = db.StringSetRange(key, 5, value); + ClassicAssert.AreEqual("15", resp.ToString()); + resp = db.StringGet(key); + ClassicAssert.AreEqual("012340123456789", resp.ToString()); + + // should update the etag internally + var updatedEtagRes = db.Execute("GETWITHETAG", key); + ClassicAssert.AreEqual(2, long.Parse(updatedEtagRes[0].ToString())); + + ClassicAssert.IsTrue(db.KeyDelete(key)); + + // new key, length 10, offset -1 -> RedisServerException ("ERR offset is out of range") + try + { + db.StringSetRange(key, -1, value); + Assert.Fail(); + } + catch (RedisServerException ex) + { + ClassicAssert.AreEqual(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_GENERIC_OFFSETOUTOFRANGE), ex.Message); + } + + // existing key, length 10, offset 0, value length 5 -> 10 ("ABCDE56789") + db.Execute("SET", key, value, "WITHETAG"); + + resp = db.StringSetRange(key, 0, newValue); + ClassicAssert.AreEqual("10", resp.ToString()); + resp = db.StringGet(key); + ClassicAssert.AreEqual("ABCDE56789", resp.ToString()); + + // should update the etag internally + updatedEtagRes = db.Execute("GETWITHETAG", key); + ClassicAssert.AreEqual(2, long.Parse(updatedEtagRes[0].ToString())); + + ClassicAssert.IsTrue(db.KeyDelete(key)); + + // key, length 10, offset 5, value length 5 -> 10 ("01234ABCDE") + db.Execute("SET", key, value, "WITHETAG"); + + resp = db.StringSetRange(key, 5, newValue); + ClassicAssert.AreEqual("10", resp.ToString()); + + updatedEtagRes = db.Execute("GETWITHETAG", key); + ClassicAssert.AreEqual(2, long.Parse(updatedEtagRes[0].ToString())); + + resp = db.StringGet(key); + ClassicAssert.AreEqual("01234ABCDE", resp.ToString()); + ClassicAssert.IsTrue(db.KeyDelete(key)); + + // existing key, length 10, offset 10, value length 5 -> 15 ("0123456789ABCDE") + db.Execute("SET", [key, value, "WITHETAG"]); + resp = db.StringSetRange(key, 10, newValue); + ClassicAssert.AreEqual("15", resp.ToString()); + resp = db.StringGet(key); + ClassicAssert.AreEqual("0123456789ABCDE", resp.ToString()); + ClassicAssert.IsTrue(db.KeyDelete(key)); + + // existing key, length 10, offset 15, value length 5 -> 20 ("0123456789\0\0\0\0\0ABCDE") + db.Execute("SET", [key, value, "WITHETAG"]); + + resp = db.StringSetRange(key, 15, newValue); + ClassicAssert.AreEqual("20", resp.ToString()); + resp = db.StringGet(key); + ClassicAssert.AreEqual("0123456789\0\0\0\0\0ABCDE", resp.ToString()); + ClassicAssert.IsTrue(db.KeyDelete(key)); + + // existing key, length 10, offset -1, value length 5 -> RedisServerException ("ERR offset is out of range") + db.Execute("SET", [key, value, "WITHETAG"]); + try + { + db.StringSetRange(key, -1, newValue); + Assert.Fail(); + } + catch (RedisServerException ex) + { + ClassicAssert.AreEqual(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_GENERIC_OFFSETOUTOFRANGE), ex.Message); + } + } + + [Test] + public void KeepTtlTestForDataInitiallySET() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + int expire = 3; + var keyA = "keyA"; + var keyB = "keyB"; + db.Execute("SET", [keyA, keyA]); + db.Execute("SET", [keyB, keyB]); + + db.KeyExpire(keyA, TimeSpan.FromSeconds(expire)); + db.KeyExpire(keyB, TimeSpan.FromSeconds(expire)); + + db.StringSet(keyA, keyA, keepTtl: true); + var time = db.KeyTimeToLive(keyA); + ClassicAssert.IsTrue(time.Value.Ticks > 0); + + db.StringSet(keyB, keyB, keepTtl: false); + time = db.KeyTimeToLive(keyB); + ClassicAssert.IsTrue(time == null); + + Thread.Sleep(expire * 1000 + 100); + + string value = db.StringGet(keyB); + ClassicAssert.AreEqual(keyB, value); + + value = db.StringGet(keyA); + ClassicAssert.AreEqual(null, value); + } + + [Test] + public void StrlenTestOnEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("SET", ["mykey", "foo bar", "WITHETAG"]); + + ClassicAssert.AreEqual(7, db.StringLength("mykey")); + ClassicAssert.AreEqual(0, db.StringLength("nokey")); + + var etagToCheck = db.Execute("GETWITHETAG", "mykey"); + ClassicAssert.AreEqual(1, long.Parse(etagToCheck[0].ToString())); + } + + [Test] + public void TTLTestMillisecondsForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "myKey"; + var val = "myKeyValue"; + var expireTimeInMilliseconds = 3000; + + var pttl = db.Execute("PTTL", key); + ClassicAssert.AreEqual(-2, (int)pttl); + + db.Execute("SET", [key, val, "WITHETAG"]); + + pttl = db.Execute("PTTL", key); + ClassicAssert.AreEqual(-1, (int)pttl); + + db.KeyExpire(key, TimeSpan.FromMilliseconds(expireTimeInMilliseconds)); + + //check TTL of the key in milliseconds + pttl = db.Execute("PTTL", key); + + ClassicAssert.IsTrue(long.TryParse(pttl.ToString(), out var pttlInMs)); + ClassicAssert.IsTrue(pttlInMs > 0); + + db.KeyPersist(key); + Thread.Sleep(expireTimeInMilliseconds); + + var _val = db.StringGet(key); + ClassicAssert.AreEqual(val, _val.ToString()); + + var ttl = db.KeyTimeToLive(key); + ClassicAssert.IsNull(ttl); + + // nothing should have affected the etag in the above commands + long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(1, etagToCheck); + } + + [Test] + public void GetDelTestForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "myKey"; + var val = "myKeyValue"; + + // Key Setup + db.Execute("SET", [key, val, "WITHETAG"]); + + var retval = db.StringGet(key); + ClassicAssert.AreEqual(val, retval.ToString()); + + retval = db.StringGetDelete(key); + ClassicAssert.AreEqual(val, retval.ToString()); + + // Try retrieving already deleted key + retval = db.StringGetDelete(key); + ClassicAssert.AreEqual(string.Empty, retval.ToString()); + + // Try retrieving & deleting non-existent key + retval = db.StringGetDelete("nonExistentKey"); + ClassicAssert.AreEqual(string.Empty, retval.ToString()); + + // Key setup with metadata + key = "myKeyWithMetadata"; + val = "myValueWithMetadata"; + + db.Execute("SET", [key, val, "WITHETAG"]); + db.KeyExpire(key, TimeSpan.FromSeconds(10000)); + + retval = db.StringGet(key); + ClassicAssert.AreEqual(val, retval.ToString()); + + retval = db.StringGetDelete(key); + ClassicAssert.AreEqual(val, retval.ToString()); + + // Try retrieving already deleted key with metadata + retval = db.StringGetDelete(key); + ClassicAssert.AreEqual(string.Empty, retval.ToString()); + } + + [Test] + public void AppendTestForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "myKey"; + var val = "myKeyValue"; + var val2 = "myKeyValue2"; + + db.Execute("SET", [key, val, "WITHETAG"]); + + var len = db.StringAppend(key, val2); + ClassicAssert.AreEqual(val.Length + val2.Length, len); + + var _val = db.StringGet(key); + ClassicAssert.AreEqual(val + val2, _val.ToString()); + + long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(2, etagToCheck); + + db.KeyDelete(key); + + // Test appending an empty string + db.Execute("SET", [key, val, "WITHETAG"]); + + var len1 = db.StringAppend(key, ""); + ClassicAssert.AreEqual(val.Length, len1); + + _val = db.StringGet(key); + ClassicAssert.AreEqual(val, _val.ToString()); + + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + // we appended nothing so this remains 1 + ClassicAssert.AreEqual(1, etagToCheck); + + // Test appending to a non-existent key + var nonExistentKey = "nonExistentKey"; + var len2 = db.StringAppend(nonExistentKey, val2); + ClassicAssert.AreEqual(val2.Length, len2); + + _val = db.StringGet(nonExistentKey); + ClassicAssert.AreEqual(val2, _val.ToString()); + + db.KeyDelete(key); + + // Test appending to a key with a large value + var largeVal = new string('a', 1000000); + db.Execute("SET", [key, largeVal, "WITHETAG"]); + var len3 = db.StringAppend(key, val2); + ClassicAssert.AreEqual(largeVal.Length + val2.Length, len3); + + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(2, etagToCheck); + + // Test appending to a key with metadata + var keyWithMetadata = "keyWithMetadata"; + db.Execute("SET", [keyWithMetadata, val, "WITHETAG"]); + db.KeyExpire(keyWithMetadata, TimeSpan.FromSeconds(10000)); + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [keyWithMetadata]))[0].ToString()); + ClassicAssert.AreEqual(1, etagToCheck); + + var len4 = db.StringAppend(keyWithMetadata, val2); + ClassicAssert.AreEqual(val.Length + val2.Length, len4); + + _val = db.StringGet(keyWithMetadata); + ClassicAssert.AreEqual(val + val2, _val.ToString()); + + var time = db.KeyTimeToLive(keyWithMetadata); + ClassicAssert.IsTrue(time.Value.TotalSeconds > 0); + + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [keyWithMetadata]))[0].ToString()); + ClassicAssert.AreEqual(2, etagToCheck); + } + + [Test] + public void SetBitOperationsOnEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string key = "miki"; + // 64 BIT BITMAP + Byte[] initialBitmap = new byte[8]; + string bitMapAsStr = Encoding.UTF8.GetString(initialBitmap); ; + + db.Execute("SET", [key, bitMapAsStr, "WITHETAG"]); + + long setbits = db.StringBitCount(key); + ClassicAssert.AreEqual(0, setbits); + + long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(1, etagToCheck); + + // set all 64 bits one by one + long expectedBitCount = 0; + long expectedEtag = 1; + for (int i = 0; i < 64; i++) + { + // SET the ith bit in the bitmap + bool originalValAtBit = db.StringSetBit(key, i, true); + ClassicAssert.IsFalse(originalValAtBit); + + expectedBitCount++; + expectedEtag++; + + bool currentBitVal = db.StringGetBit(key, i); + ClassicAssert.IsTrue(currentBitVal); + + setbits = db.StringBitCount(key); + ClassicAssert.AreEqual(expectedBitCount, setbits); + + // Use BitPosition to find the first set bit + long firstSetBitPosition = db.StringBitPosition(key, true); + ClassicAssert.AreEqual(0, firstSetBitPosition); // As we are setting bits in order, first set bit should be 0 + + // find the first unset bit + long firstUnsetBitPos = db.StringBitPosition(key, false); + long firstUnsetBitPosExpected = i == 63 ? -1 : i + 1; + ClassicAssert.AreEqual(firstUnsetBitPosExpected, firstUnsetBitPos); // As we are setting bits in order, first unset bit should be 1 ahead + + + // with each bit set that we do, we are increasing the etag as well by 1 + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(expectedEtag, etagToCheck); + } + + // unset all 64 bits one by one in reverse order + for (int i = 63; i > -1; i--) + { + bool originalValAtBit = db.StringSetBit(key, i, false); + ClassicAssert.IsTrue(originalValAtBit); + + expectedEtag++; + expectedBitCount--; + + bool currentBitVal = db.StringGetBit(key, i); + ClassicAssert.IsFalse(currentBitVal); + + setbits = db.StringBitCount(key); + ClassicAssert.AreEqual(expectedBitCount, setbits); + + // find the first set bit + long firstSetBit = db.StringBitPosition(key, true); + long expectedSetBit = i == 0 ? -1 : 0; + ClassicAssert.AreEqual(expectedSetBit, firstSetBit); + + // Use BitPosition to find the first unset bit + long firstUnsetBitPosition = db.StringBitPosition(key, false); + ClassicAssert.AreEqual(i, firstUnsetBitPosition); // After unsetting, the first unset bit should be i + + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(expectedEtag, etagToCheck); + } + } + + [Test] + public void BitFieldSetGetOnEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "mewo"; + + // Arrange - Set an 8-bit unsigned value at offset 0 + db.Execute("SET", [key, Encoding.UTF8.GetString(new byte[1]), "WITHETAG"]); // Initialize key with an empty byte + + // Act - Set value to 127 (binary: 01111111) + db.Execute("BITFIELD", key, "SET", "u8", "0", "127"); + + long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(2, etagToCheck); + + // Get value back + var getResult = (RedisResult[])db.Execute("BITFIELD", key, "GET", "u8", "0"); + + // Assert + ClassicAssert.AreEqual(127, (long)getResult[0]); // Ensure the value set was retrieved correctly + } + + [Test] + public void BitFieldIncrementWithWrapOverflowOnEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "mewo"; + + // Arrange - Set an 8-bit unsigned value at offset 0 + db.Execute("SET", [key, Encoding.UTF8.GetString(new byte[1]), "WITHETAG"]); // Initialize key with an empty byte + + // Act - Set initial value to 255 and try to increment by 1 + db.Execute("BITFIELD", key, "SET", "u8", "0", "255"); + long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(2, etagToCheck); + + var incrResult = db.Execute("BITFIELD", key, "INCRBY", "u8", "0", "1"); + + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(3, etagToCheck); + + // Assert + ClassicAssert.AreEqual(0, (long)incrResult); // Should wrap around and return 0 + } + + [Test] + public void BitFieldIncrementWithSaturateOverflowOnEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "mewo"; + + // Arrange - Set an 8-bit unsigned value at offset 0 + db.Execute("SET", [key, Encoding.UTF8.GetString(new byte[1]), "WITHETAG"]); // Initialize key with an empty byte + + // Act - Set initial value to 250 and try to increment by 10 with saturate overflow + var bitfieldRes = db.Execute("BITFIELD", key, "SET", "u8", "0", "250"); + ClassicAssert.AreEqual(0, (long)bitfieldRes); + + var result = (RedisResult[])db.Execute("GETWITHETAG", [key]); + long etagToCheck = long.Parse(result[0].ToString()); + ClassicAssert.AreEqual(2, etagToCheck); + + var incrResult = db.Execute("BITFIELD", key, "OVERFLOW", "SAT", "INCRBY", "u8", "0", "10"); + + etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); + ClassicAssert.AreEqual(3, etagToCheck); + + // Assert + ClassicAssert.AreEqual(255, (long)incrResult); // Should saturate at the max value of 255 for u8 + } + + [Test] + public void HyperLogLogCommandsShouldReturnWrongTypeErrorForEtagSetData() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "mewo"; + var key2 = "dude"; + + db.Execute("SET", [key, "mars", "WITHETAG"]); + db.Execute("SET", [key2, "marsrover", "WITHETAG"]); + + RedisServerException ex = Assert.Throws(() => db.Execute("PFADD", [key, "woohoo"])); + + ClassicAssert.IsNotNull(ex); + ClassicAssert.AreEqual(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE_HLL), ex.Message); + + ex = Assert.Throws(() => db.Execute("PFMERGE", [key, key2])); + + ClassicAssert.IsNotNull(ex); + ClassicAssert.AreEqual(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE_HLL), ex.Message); + } + + [Test] + public void SetWithWITHETAGOnANewUpsertWillCreateKeyValueWithoutEtag() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + string key = "mickey"; + string val = "mouse"; + + // a new upsert on a non-existing key will retain the "nil" etag + db.Execute("SET", [key, val, "WITHETAG"]).ToString(); + + RedisResult[] res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + RedisResult etag = res[0]; + string value = res[1].ToString(); + + ClassicAssert.AreEqual("1", etag.ToString()); + ClassicAssert.AreEqual(val, value); + + string newval = "clubhouse"; + + // a new upsert on an existing key will reset the etag on the key + db.Execute("SET", [key, newval]).ToString(); + res = (RedisResult[])db.Execute("GETWITHETAG", [key]); + etag = res[0]; + value = res[1].ToString(); + + ClassicAssert.AreEqual("0", etag.ToString()); + ClassicAssert.AreEqual(newval, value); + } + + #endregion + } +} \ No newline at end of file diff --git a/test/Garnet.test/RespTests.cs b/test/Garnet.test/RespTests.cs index 4ad3363534..62127deb27 100644 --- a/test/Garnet.test/RespTests.cs +++ b/test/Garnet.test/RespTests.cs @@ -1848,6 +1848,48 @@ public void SingleRenameNxWithOldKeyAndNewKeyAsSame() ClassicAssert.AreEqual(origValue, retValue); } + [Test] + public void SingleRenameNXWithEtagSetOldAndNewKey() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var origValue = "test1"; + var key = "key1"; + var newKey = "key2"; + + db.Execute("SET", key, origValue, "WITHETAG"); + db.Execute("SET", newKey, "foo", "WITHETAG"); + + var result = db.KeyRename(key, newKey, When.NotExists); + ClassicAssert.IsFalse(result); + } + + [Test] + public void SingleRenameNXWithEtagSetOldKey() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var origValue = "test1"; + var key = "key1"; + var newKey = "key2"; + + db.Execute("SET", key, origValue, "WITHETAG"); + + var result = db.KeyRename(key, newKey, When.NotExists); + ClassicAssert.IsTrue(result); + + string retValue = db.StringGet(newKey); + ClassicAssert.AreEqual(origValue, retValue); + + var oldKeyRes = db.StringGet(key); + ClassicAssert.IsTrue(oldKeyRes.IsNull); + + // Since the original key was set with etag, the new key should have an etag attached to it + var etagRes = (RedisResult[])db.Execute("GETWITHETAG", newKey); + ClassicAssert.AreEqual(0, (long)etagRes[0]); + ClassicAssert.AreEqual(origValue, etagRes[1].ToString()); + } + #endregion [Test] diff --git a/test/Garnet.test/TransactionTests.cs b/test/Garnet.test/TransactionTests.cs index 76e91799b1..aefd400cdc 100644 --- a/test/Garnet.test/TransactionTests.cs +++ b/test/Garnet.test/TransactionTests.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System; +using System.Text; using System.Threading.Tasks; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -212,10 +213,72 @@ public async Task SimpleWatchTest() res = lightClientRequest.SendCommand("EXEC"); expectedResponse = "*2\r\n$14\r\nvalue1_updated\r\n+OK\r\n"; - ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + ClassicAssert.AreEqual( + expectedResponse, + Encoding.ASCII.GetString(res.AsSpan().Slice(0, expectedResponse.Length))); } + [Test] + public async Task WatchTestWithSetWithEtag() + { + var lightClientRequest = TestUtils.CreateRequest(); + byte[] res; + + string expectedResponse = ":1\r\n"; + res = lightClientRequest.SendCommand("SET key1 value1 WITHETAG"); + var debug = res.AsSpan().Slice(0, expectedResponse.Length).ToArray(); + ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + + expectedResponse = "+OK\r\n"; + res = lightClientRequest.SendCommand("WATCH key1"); + ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + + res = lightClientRequest.SendCommand("MULTI"); + ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + + res = lightClientRequest.SendCommand("GET key1"); + expectedResponse = "+QUEUED\r\n"; + ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + res = lightClientRequest.SendCommand("SET key2 value2"); + ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + + await Task.Run(() => + { + using var lightClientRequestCopy = TestUtils.CreateRequest(); + string command = "SET key1 value1_updated WITHETAG"; + lightClientRequestCopy.SendCommand(command); + }); + + res = lightClientRequest.SendCommand("EXEC"); + expectedResponse = "*-1"; + ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); + + // This one should Commit + lightClientRequest.SendCommand("MULTI"); + + lightClientRequest.SendCommand("GET key1"); + lightClientRequest.SendCommand("SET key2 value2"); + // check that all the etag commands can be called inside a transaction + lightClientRequest.SendCommand("SET key3 value2 WITHETAG"); + lightClientRequest.SendCommand("GETWITHETAG key3"); + lightClientRequest.SendCommand("GETIFNOTMATCH key3 1"); + lightClientRequest.SendCommand("SETIFMATCH key3 anotherVal 1"); + lightClientRequest.SendCommand("SET key3 arandomval WITHETAG"); + + res = lightClientRequest.SendCommand("EXEC"); + + expectedResponse = "*7\r\n$14\r\nvalue1_updated\r\n+OK\r\n:1\r\n*2\r\n:1\r\n$6\r\nvalue2\r\n*2\r\n:1\r\n$-1\r\n*2\r\n:2\r\n$-1\r\n:3\r\n"; + string response = Encoding.ASCII.GetString(res.AsSpan().Slice(0, expectedResponse.Length)); + ClassicAssert.AreEqual(expectedResponse, response); + + // check if we still have the appropriate etag on the key we had set + var otherLighClientRequest = TestUtils.CreateRequest(); + res = otherLighClientRequest.SendCommand("GETWITHETAG key1"); + expectedResponse = "*2\r\n:2\r\n$14\r\nvalue1_updated\r\n"; + response = Encoding.ASCII.GetString(res.AsSpan().Slice(0, expectedResponse.Length)); + ClassicAssert.AreEqual(response, expectedResponse); + } [Test] public async Task WatchNonExistentKey() @@ -307,7 +370,8 @@ public async Task WatchKeyFromDisk() private static void updateKey(string key, string value) { using var lightClientRequest = TestUtils.CreateRequest(); - byte[] res = lightClientRequest.SendCommand("SET " + key + " " + value); + string command = $"SET {key} {value}"; + byte[] res = lightClientRequest.SendCommand(command); string expectedResponse = "+OK\r\n"; ClassicAssert.AreEqual(res.AsSpan().Slice(0, expectedResponse.Length).ToArray(), expectedResponse); } diff --git a/website/docs/commands/garnet-specific.md b/website/docs/commands/garnet-specific.md index d98b6bcfa0..1a31e55d29 100644 --- a/website/docs/commands/garnet-specific.md +++ b/website/docs/commands/garnet-specific.md @@ -122,3 +122,92 @@ initialization code registers all relevant commands and transactions automatical for details. --- + +## Native ETag Support + +Garnet provides support for ETags on raw strings. By using the ETag-related commands outlined below, you can associate any **string based key-value pair** inserted into Garnet with an automatically updated ETag. + +Compatibility with non-ETag commands and the behavior of data inserted with ETags are detailed at the end of this document. + +To initialize a key value pair with an ETag you can use either the SET command with the newly added "WITHETAG" optional flag, or you can take any existing Key value pair and call SETIFMATCH with the ETag argument as 0 (Any key value pair without an explicit ETag has an ETag of 0 implicitly). You can read more about setting an initial ETag via SET [here](../commands/raw-string#set) + +--- + +### **GETWITHETAG** + +#### **Syntax** + +```bash +GETWITHETAG key +``` + +Retrieves the value and the ETag associated with the given key. + +#### **Response** + +One of the following: + +- **Array reply**: An array of two items returned on success. The first item is an integer representing the ETag, and the second is the bulk string value of the key. If called on a key-value pair without ETag, the etag will be 0. +- **Nil reply**: If the key does not exist. + +--- + +### **SETIFMATCH** + +#### **Syntax** + +```bash +SETIFMATCH key value etag [EX seconds | PX milliseconds] +``` + +Updates the value of a key if the provided ETag matches the current ETag of the key. + +**Options:** +* EX seconds -- Set the specified expire time, in seconds (a positive integer). +* PX milliseconds -- Set the specified expire time, in milliseconds (a positive integer). + +#### **Response** + +One of the following: + +- **Array reply**: If etags match an array where the first item is the updated etag, and the second value is nil. If the etags do not match the array will hold the latest etag, and the latest value in order. +- **Nil reply**: If the key does not exist. + +--- + +### **GETIFNOTMATCH** + +#### **Syntax** + +```bash +GETIFNOTMATCH key etag +``` + +Retrieves the value if the ETag associated with the key has changed; otherwise, returns a response indicating no change. + +#### **Response** + +One of the following: + +- **Array reply**: If the ETag does not match, an array of two items is returned. The first item is the latest ETag, and the second item is the value associated with the key. If the Etag matches the first item in the response array is the etag and the second item is nil. +- **Nil reply**: If the key does not exist. + +--- + +### Compatibility and Behavior with Non-ETag Commands + +ETags are currently not supported for servers running in Cluster mode. This will be supported soon. + +Below is the expected behavior of ETag-associated key-value pairs when non-ETag commands are used. + +- **MSET, BITOP**: These commands will replace an existing ETag-associated key-value pair with a non-ETag key-value pair, effectively removing the ETag. + +- **SET**: Only if used with additional option "WITHETAG" will calling SET update the etag while inserting the new key-value pair over the existing key-value pair. + +- **RENAME**: RENAME takes an option for WITHETAG. When called WITHETAG it will rename the key with an etag if the key being renamed to did not exist, else it will increment the existing etag of the key being renamed to. + +- **Custom Commands**: While etag based key value pairs **can be used blindly inside of custom transactions and custom procedures**, ETag set key value pairs are **not supported to be used from inside of Custom Raw String Functions.** + +All other commands will update the etag internally if they modify the underlying data, and any responses from them will not expose the etag to the client. To the users the etag and it's updates remain hidden in non-etag commands. + +--- diff --git a/website/docs/commands/generic-commands.md b/website/docs/commands/generic-commands.md index 59ebf9b933..27c8ae6946 100644 --- a/website/docs/commands/generic-commands.md +++ b/website/docs/commands/generic-commands.md @@ -366,11 +366,14 @@ One of the following: #### Syntax ```bash - RENAME key newkey + RENAME key newkey [WITHETAG] ``` Renames key to newkey. It returns an error when key does not exist. If newkey already exists it is overwritten, when this happens RENAME executes an implicit [DEL](#del) operation. +#### **Options:** +* WITHETAG - If the newkey did not exist, the newkey will now have an ETag associated with it after the rename. If the newkey existed before with an ETag the RENAME will update the ETag. If the newkey existed before without an ETag, then after the RENAME the newkey would have an ETag associated with it. You can read more about ETags [here](../commands/garnet-specific-commands#native-etag-support). + #### Resp Reply Simple string reply: OK. @@ -382,11 +385,14 @@ Simple string reply: OK. #### Syntax ```bash - RENAME key newkey + RENAMENX key newkey [WITHETAG] ``` Renames key to newkey if newkey does not yet exist. It returns an error when key does not exist. +#### **Options:** +* WITHETAG - The newkey will now have an ETag associated with it after the rename. You can read more about ETags [here](../commands/garnet-specific-commands#native-etag-support). + #### Resp Reply One of the following: diff --git a/website/docs/commands/raw-string.md b/website/docs/commands/raw-string.md index 47d3314489..4d8f99c019 100644 --- a/website/docs/commands/raw-string.md +++ b/website/docs/commands/raw-string.md @@ -316,7 +316,7 @@ Simple string reply: OK. #### Syntax ```bash - SET key value [NX | XX] [GET] [EX seconds | PX milliseconds | + SET key value [NX | XX] [GET] [EX seconds | PX milliseconds] [KEEPTTL] [WITHETAG] ``` Set **key** to hold the string value. If key already holds a value, it is overwritten, regardless of its type. Any previous time to live associated with the **key** is discarded on successful SET operation. @@ -328,6 +328,7 @@ Set **key** to hold the string value. If key already holds a value, it is overwr * NX -- Only set the key if it does not already exist. * XX -- Only set the key if it already exists. * KEEPTTL -- Retain the time to live associated with the key. +* WITHETAG -- Adding this option sets the Key Value pair with an initial ETag, if called on an existing key value pair with an ETag, this command will update the ETag transparently. This is a Garnet specific command, you can read more about ETag support [here](../commands/garnet-specific-commands#native-etag-support). WITHETAG and GET options cannot be sent at the same time. #### Resp Reply @@ -337,6 +338,7 @@ Any of the following: * Simple string reply: OK. GET not given: The key was set. * Nil reply: GET given: The key didn't exist before the SET. * Bulk string reply: GET given: The previous value of the key. +* Integer reply: WITHETAG given: The ETag either created on the value, or the updated Etag. ---