Skip to content

Commit

Permalink
- Replaced Free with more conventional IDisposable on `ICustomSam…
Browse files Browse the repository at this point in the history
…pler`

 - Improved documentation on `Apply` related to setting the `Sorted` flag
  • Loading branch information
martindevans committed Oct 25, 2024
1 parent a88ecba commit 6b10049
Showing 1 changed file with 9 additions and 7 deletions.
16 changes: 9 additions & 7 deletions LLama/Native/SafeLLamaSamplerHandle.cs
Original file line number Diff line number Diff line change
Expand Up @@ -701,23 +701,30 @@ private static unsafe void Free(ref LLamaSamplerNative smpl)

sampler._gcHandle.Free();

sampler._sampler.Free();
sampler._sampler.Dispose();
}
}

/// <summary>
/// A custom sampler stage for modifying logits or selecting a token
/// </summary>
public interface ICustomSampler
: IDisposable
{
/// <summary>
/// The human readable name of this stage
/// </summary>
string Name { get; }

/// <summary>
/// Apply this stage to a set of logits. This can modify logits or select a token (or both). If logits are modified, the Sorted flag must be cleared.
/// Apply this stage to a set of logits.
/// This can modify logits or select a token (or both).
/// If logits are modified the Sorted flag <b>must</b> be set to false.
/// </summary>
/// <remarks>
/// If the logits are no longer sorted after the custom sampler has run it is <b>critically</b> important to
/// set <i>Sorted=false</i>. If unsure, always set it to false, this is a safe default.
/// </remarks>
/// <param name="tokenData"></param>
void Apply(ref LLamaTokenDataArrayNative tokenData);

Expand All @@ -736,9 +743,4 @@ public interface ICustomSampler
/// Create a clone of this sampler
/// </summary>
ICustomSampler Clone();

/// <summary>
/// Free all unmanaged resources held by this sampler
/// </summary>
void Free();
}

0 comments on commit 6b10049

Please sign in to comment.