[frontend][LLM] Assign SEP to all the tail elements.

buddy-compiler · Aug 29, 2023 · 19fb275 · 19fb275
1 parent b572cd6
commit 19fb275
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 1 deletion.
diff --git a/frontend/Interfaces/buddy/LLM/TextContainer.h b/frontend/Interfaces/buddy/LLM/TextContainer.h
@@ -125,7 +125,10 @@ void Text<T, N>::tokenize(const std::string &vocab, long long length) {
     processToken(token, tokenCnt);
     token.clear();
   }
-  this->aligned[tokenCnt] = 102; // [SEP] NLP Separator Marker
+  // [SEP] NLP Separator Marker
+  for (long long i = tokenCnt; i < length; i++) {
+    this->aligned[i] = 102;
+  }
 }
 
 template <typename T, size_t N>

diff --git a/tests/Interface/core/TextContainerTest.cpp b/tests/Interface/core/TextContainerTest.cpp
@@ -57,6 +57,12 @@ int main() {
   fprintf(stderr, "%lld\n", pureStrContainer.getData()[7]);
   // CHECK: 102
   fprintf(stderr, "%lld\n", pureStrContainer.getData()[8]);
+	// CHECK: 102
+  fprintf(stderr, "%lld\n", pureStrContainer.getData()[9]);
+	// CHECK: 102
+  fprintf(stderr, "%lld\n", pureStrContainer.getData()[10]);
+	// CHECK: 102
+  fprintf(stderr, "%lld\n", pureStrContainer.getData()[11]);
 
   //===--------------------------------------------------------------------===//
   // Test text constructor for punctuation.
@@ -84,6 +90,10 @@ int main() {
   fprintf(stderr, "%lld\n", puncStrContainer.getData()[8]);
   // CHECK: 102
   fprintf(stderr, "%lld\n", puncStrContainer.getData()[9]);
+	// CHECK: 102
+  fprintf(stderr, "%lld\n", puncStrContainer.getData()[10]);
+	// CHECK: 102
+  fprintf(stderr, "%lld\n", puncStrContainer.getData()[11]);
 
   //===--------------------------------------------------------------------===//
   // Test text constructor for corner cases.
@@ -113,4 +123,6 @@ int main() {
   fprintf(stderr, "%lld\n", cornerStrContainer.getData()[9]);
   // CHECK: 102
   fprintf(stderr, "%lld\n", cornerStrContainer.getData()[10]);
+	// CHECK: 102
+  fprintf(stderr, "%lld\n", cornerStrContainer.getData()[11]);
 }