diff --git a/src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/ICUTokenizer.cs b/src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/ICUTokenizer.cs index 2b37cde955..7b31755118 100644 --- a/src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/ICUTokenizer.cs +++ b/src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/ICUTokenizer.cs @@ -57,6 +57,8 @@ public sealed class ICUTokenizer : Tokenizer private readonly ITypeAttribute typeAtt; private readonly IScriptAttribute scriptAtt; + private static readonly object syncLock = new object(); // LUCENENET specific - workaround until BreakIterator is made thread safe (LUCENENET TODO: TO REVERT) + /// /// Construct a new that breaks text into words from the given /// . @@ -109,23 +111,27 @@ public ICUTokenizer(AttributeFactory factory, TextReader input, ICUTokenizerConf public override bool IncrementToken() { - ClearAttributes(); - if (length == 0) - Refill(); - while (!IncrementTokenBuffer()) + lock (syncLock) { - Refill(); - if (length <= 0) // no more bytes to read; - return false; + ClearAttributes(); + if (length == 0) + Refill(); + while (!IncrementTokenBuffer()) + { + Refill(); + if (length <= 0) // no more bytes to read; + return false; + } + return true; } - return true; } public override void Reset() { base.Reset(); - breaker.SetText(buffer, 0, 0); + lock (syncLock) + breaker.SetText(buffer, 0, 0); length = usableLength = offset = 0; } @@ -187,7 +193,8 @@ private void Refill() */ } - breaker.SetText(buffer, 0, Math.Max(0, usableLength)); + lock (syncLock) + breaker.SetText(buffer, 0, Math.Max(0, usableLength)); } // TODO: refactor to a shared readFully somewhere @@ -236,7 +243,7 @@ private bool IncrementTokenBuffer() offsetAtt.SetOffset(CorrectOffset(offset + start), CorrectOffset(offset + end)); typeAtt.Type = config.GetType(breaker.ScriptCode, breaker.RuleStatus); scriptAtt.Code = breaker.ScriptCode; - + return true; } }