diff --git a/classifier.go b/classifier.go
index 39b6ea73..ab50ee83 100644
--- a/classifier.go
+++ b/classifier.go
@@ -2,15 +2,15 @@ package enry
import (
"math"
+ "sort"
"gopkg.in/src-d/enry.v1/internal/tokenizer"
)
-// Classifier is the interface that contains the method Classify which is in charge to assign scores to the possibles candidates.
-// The scores must order the candidates so as the highest score be the most probably language of the content. The candidates is
-// a map which can be used to assign weights to languages dynamically.
+// Classifier is the interface in charge to detect the possible languages of the given content based on a set of
+// candidates. Candidates is a map which can be used to assign weights to languages dynamically.
type Classifier interface {
- Classify(content []byte, candidates map[string]float64) map[string]float64
+ Classify(content []byte, candidates map[string]float64) (languages []string)
}
type classifier struct {
@@ -19,7 +19,13 @@ type classifier struct {
tokensTotal float64
}
-func (c *classifier) Classify(content []byte, candidates map[string]float64) map[string]float64 {
+type scoredLanguage struct {
+ language string
+ score float64
+}
+
+// Classify returns a sorted slice of possible languages sorted by decreasing language's probability
+func (c *classifier) Classify(content []byte, candidates map[string]float64) []string {
if len(content) == 0 {
return nil
}
@@ -31,18 +37,35 @@ func (c *classifier) Classify(content []byte, candidates map[string]float64) map
languages = make(map[string]float64, len(candidates))
for candidate, weight := range candidates {
if lang, ok := GetLanguageByAlias(candidate); ok {
- languages[lang] = weight
+ candidate = lang
}
+
+ languages[candidate] = weight
}
}
tokens := tokenizer.Tokenize(content)
- scores := make(map[string]float64, len(languages))
+ scoredLangs := make([]*scoredLanguage, 0, len(languages))
for language := range languages {
- scores[language] = c.tokensLogProbability(tokens, language) + c.languagesLogProbabilities[language]
+ scoredLang := &scoredLanguage{
+ language: language,
+ score: c.tokensLogProbability(tokens, language) + c.languagesLogProbabilities[language],
+ }
+
+ scoredLangs = append(scoredLangs, scoredLang)
+ }
+
+ return sortLanguagesByScore(scoredLangs)
+}
+
+func sortLanguagesByScore(scoredLangs []*scoredLanguage) []string {
+ sort.SliceStable(scoredLangs, func(i, j int) bool { return scoredLangs[j].score < scoredLangs[i].score })
+ sortedLanguages := make([]string, 0, len(scoredLangs))
+ for _, scoredLang := range scoredLangs {
+ sortedLanguages = append(sortedLanguages, scoredLang.language)
}
- return scores
+ return sortedLanguages
}
func (c *classifier) knownLangs() map[string]float64 {
diff --git a/common.go b/common.go
index a4615e4a..dd6f4b55 100644
--- a/common.go
+++ b/common.go
@@ -1,150 +1,398 @@
package enry
import (
- "math"
+ "bufio"
+ "bytes"
"path/filepath"
+ "regexp"
"strings"
)
// OtherLanguage is used as a zero value when a function can not return a specific language.
-const OtherLanguage = "Other"
+const OtherLanguage = ""
// Strategy type fix the signature for the functions that can be used as a strategy.
-type Strategy func(filename string, content []byte) (languages []string)
+type Strategy func(filename string, content []byte, candidates []string) (languages []string)
-var strategies = []Strategy{
+// DefaultStrategies is the strategies' sequence GetLanguage uses to detect languages.
+var DefaultStrategies = []Strategy{
GetLanguagesByModeline,
GetLanguagesByFilename,
GetLanguagesByShebang,
GetLanguagesByExtension,
GetLanguagesByContent,
+ GetLanguagesByClassifier,
}
// GetLanguage applies a sequence of strategies based on the given filename and content
// to find out the most probably language to return.
-func GetLanguage(filename string, content []byte) string {
- candidates := map[string]float64{}
- for _, strategy := range strategies {
- languages := strategy(filename, content)
+func GetLanguage(filename string, content []byte) (language string) {
+ languages := GetLanguages(filename, content)
+ return firstLanguage(languages)
+}
+
+func firstLanguage(languages []string) string {
+ if len(languages) == 0 {
+ return OtherLanguage
+ }
+
+ return languages[0]
+}
+
+// GetLanguageByModeline returns detected language. If there are more than one possibles languages
+// it returns the first language by alphabetically order and safe to false.
+func GetLanguageByModeline(content []byte) (language string, safe bool) {
+ return getLanguageByStrategy(GetLanguagesByModeline, "", content, nil)
+}
+
+// GetLanguageByEmacsModeline returns detected language. If there are more than one possibles languages
+// it returns the first language by alphabetically order and safe to false.
+func GetLanguageByEmacsModeline(content []byte) (language string, safe bool) {
+ return getLanguageByStrategy(GetLanguagesByEmacsModeline, "", content, nil)
+}
+
+// GetLanguageByVimModeline returns detected language. If there are more than one possibles languages
+// it returns the first language by alphabetically order and safe to false.
+func GetLanguageByVimModeline(content []byte) (language string, safe bool) {
+ return getLanguageByStrategy(GetLanguagesByVimModeline, "", content, nil)
+}
+
+// GetLanguageByFilename returns detected language. If there are more than one possibles languages
+// it returns the first language by alphabetically order and safe to false.
+func GetLanguageByFilename(filename string) (language string, safe bool) {
+ return getLanguageByStrategy(GetLanguagesByFilename, filename, nil, nil)
+}
+
+// GetLanguageByShebang returns detected language. If there are more than one possibles languages
+// it returns the first language by alphabetically order and safe to false.
+func GetLanguageByShebang(content []byte) (language string, safe bool) {
+ return getLanguageByStrategy(GetLanguagesByShebang, "", content, nil)
+}
+
+// GetLanguageByExtension returns detected language. If there are more than one possibles languages
+// it returns the first language by alphabetically order and safe to false.
+func GetLanguageByExtension(filename string) (language string, safe bool) {
+ return getLanguageByStrategy(GetLanguagesByExtension, filename, nil, nil)
+}
+
+// GetLanguageByContent returns detected language. If there are more than one possibles languages
+// it returns the first language by alphabetically order and safe to false.
+func GetLanguageByContent(content []byte) (language string, safe bool) {
+ return getLanguageByStrategy(GetLanguagesByContent, "", content, nil)
+}
+
+// GetLanguageByClassifier returns the most probably language detected for the given content. It uses
+// DefaultClassifier, if no candidates are provided it returns OtherLanguage.
+func GetLanguageByClassifier(content []byte, candidates []string) (language string, safe bool) {
+ return getLanguageByStrategy(GetLanguagesByClassifier, "", content, candidates)
+}
+
+func getLanguageByStrategy(strategy Strategy, filename string, content []byte, candidates []string) (string, bool) {
+ languages := strategy(filename, content, candidates)
+ return getFirstLanguageAndSafe(languages)
+}
+
+func getFirstLanguageAndSafe(languages []string) (language string, safe bool) {
+ language = firstLanguage(languages)
+ safe = len(languages) == 1
+ return
+}
+
+// GetLanguageBySpecificClassifier returns the most probably language for the given content using
+// classifier to detect language.
+func GetLanguageBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (language string, safe bool) {
+ languages := GetLanguagesBySpecificClassifier(content, candidates, classifier)
+ return getFirstLanguageAndSafe(languages)
+}
+
+// GetLanguages applies a sequence of strategies based on the given filename and content
+// to find out the most probably languages to return.
+func GetLanguages(filename string, content []byte) []string {
+ var languages []string
+ candidates := []string{}
+ for _, strategy := range DefaultStrategies {
+ languages = strategy(filename, content, candidates)
if len(languages) == 1 {
- return languages[0]
+ return languages
}
if len(languages) > 0 {
- for _, language := range languages {
- candidates[language]++
- }
+ candidates = append(candidates, languages...)
}
}
- if len(candidates) == 0 {
- return OtherLanguage
+ return languages
+}
+
+// GetLanguagesByModeline returns a slice of possible languages for the given content, filename will be ignored.
+// It is comply with the signature to be a Strategy type.
+func GetLanguagesByModeline(filename string, content []byte, candidates []string) []string {
+ headFoot := getHeaderAndFooter(content)
+ var languages []string
+ for _, getLang := range modelinesFunc {
+ languages = getLang("", headFoot, candidates)
+ if len(languages) > 0 {
+ break
+ }
}
- lang := GetLanguageByClassifier(content, candidates, nil)
- return lang
+ return languages
}
-// GetLanguageByModeline returns the language of the given content looking for the modeline,
-// and safe to indicate the sureness of returned language.
-func GetLanguageByModeline(content []byte) (lang string, safe bool) {
- return getLangAndSafe("", content, GetLanguagesByModeline)
+var modelinesFunc = []Strategy{
+ GetLanguagesByEmacsModeline,
+ GetLanguagesByVimModeline,
}
-// GetLanguageByFilename returns a language based on the given filename, and safe to indicate
-// the sureness of returned language.
-func GetLanguageByFilename(filename string) (lang string, safe bool) {
- return getLangAndSafe(filename, nil, GetLanguagesByFilename)
-}
+func getHeaderAndFooter(content []byte) []byte {
+ const searchScope = 5
+ if bytes.Count(content, []byte("\n")) < 2*searchScope {
+ return content
+ }
-// GetLanguagesByFilename returns a slice of possible languages for the given filename, content will be ignored.
-// It accomplish the signature to be a Strategy type.
-func GetLanguagesByFilename(filename string, content []byte) []string {
- return languagesByFilename[filename]
+ header := headScope(content, searchScope)
+ footer := footScope(content, searchScope)
+ headerAndFooter := make([]byte, 0, len(content[:header])+len(content[footer:]))
+ headerAndFooter = append(headerAndFooter, content[:header]...)
+ headerAndFooter = append(headerAndFooter, content[footer:]...)
+ return headerAndFooter
}
-// GetLanguageByShebang returns the language of the given content looking for the shebang line,
-// and safe to indicate the sureness of returned language.
-func GetLanguageByShebang(content []byte) (lang string, safe bool) {
- return getLangAndSafe("", content, GetLanguagesByShebang)
-}
+func headScope(content []byte, scope int) (index int) {
+ for i := 0; i < scope; i++ {
+ eol := bytes.IndexAny(content, "\n")
+ content = content[eol+1:]
+ index += eol
+ }
-// GetLanguageByExtension returns a language based on the given filename, and safe to indicate
-// the sureness of returned language.
-func GetLanguageByExtension(filename string) (lang string, safe bool) {
- return getLangAndSafe(filename, nil, GetLanguagesByExtension)
+ return index + scope - 1
}
-// GetLanguagesByExtension returns a slice of possible languages for the given filename, content will be ignored.
-// It accomplish the signature to be a Strategy type.
-func GetLanguagesByExtension(filename string, content []byte) []string {
- ext := strings.ToLower(filepath.Ext(filename))
- return languagesByExtension[ext]
+func footScope(content []byte, scope int) (index int) {
+ for i := 0; i < scope; i++ {
+ index = bytes.LastIndexAny(content, "\n")
+ content = content[:index]
+ }
+
+ return index + 1
}
-// GetLanguageByContent returns a language based on the filename and heuristics applies to the content,
-// and safe to indicate the sureness of returned language.
-func GetLanguageByContent(filename string, content []byte) (lang string, safe bool) {
- return getLangAndSafe(filename, content, GetLanguagesByContent)
+var (
+ reEmacsModeline = regexp.MustCompile(`.*-\*-\s*(.+?)\s*-\*-.*(?m:$)`)
+ reEmacsLang = regexp.MustCompile(`.*(?i:mode)\s*:\s*([^\s;]+)\s*;*.*`)
+ reVimModeline = regexp.MustCompile(`(?:(?m:\s|^)vi(?:m[<=>]?\d+|m)?|[\t\x20]*ex)\s*[:]\s*(.*)(?m:$)`)
+ reVimLang = regexp.MustCompile(`(?i:filetype|ft|syntax)\s*=(\w+)(?:\s|:|$)`)
+)
+
+// GetLanguagesByEmacsModeline returns a slice of possible languages for the given content, filename and candidates
+// will be ignored. It is comply with the signature to be a Strategy type.
+func GetLanguagesByEmacsModeline(filename string, content []byte, candidates []string) []string {
+ matched := reEmacsModeline.FindAllSubmatch(content, -1)
+ if matched == nil {
+ return nil
+ }
+
+ // only take the last matched line, discard previous lines
+ lastLineMatched := matched[len(matched)-1][1]
+ matchedAlias := reEmacsLang.FindSubmatch(lastLineMatched)
+ var alias string
+ if matchedAlias != nil {
+ alias = string(matchedAlias[1])
+ } else {
+ alias = string(lastLineMatched)
+ }
+
+ language, ok := GetLanguageByAlias(alias)
+ if !ok {
+ return nil
+ }
+
+ return []string{language}
}
-// GetLanguagesByContent returns a slice of possible languages for the given content, filename will be ignored.
-// It accomplish the signature to be a Strategy type.
-func GetLanguagesByContent(filename string, content []byte) []string {
- ext := strings.ToLower(filepath.Ext(filename))
- fnMatcher, ok := contentMatchers[ext]
+// GetLanguagesByVimModeline returns a slice of possible languages for the given content, filename and candidates
+// will be ignored. It is comply with the signature to be a Strategy type.
+func GetLanguagesByVimModeline(filename string, content []byte, candidates []string) []string {
+ matched := reVimModeline.FindAllSubmatch(content, -1)
+ if matched == nil {
+ return nil
+ }
+
+ // only take the last matched line, discard previous lines
+ lastLineMatched := matched[len(matched)-1][1]
+ matchedAlias := reVimLang.FindAllSubmatch(lastLineMatched, -1)
+ if matchedAlias == nil {
+ return nil
+ }
+
+ alias := string(matchedAlias[0][1])
+ if len(matchedAlias) > 1 {
+ // cases:
+ // matchedAlias = [["syntax=ruby " "ruby"] ["ft=python " "python"] ["filetype=perl " "perl"]] returns OtherLanguage;
+ // matchedAlias = [["syntax=python " "python"] ["ft=python " "python"] ["filetype=python " "python"]] returns "Python";
+ for _, match := range matchedAlias {
+ otherAlias := string(match[1])
+ if otherAlias != alias {
+ return nil
+ }
+ }
+ }
+
+ language, ok := GetLanguageByAlias(alias)
if !ok {
return nil
}
- return fnMatcher(content)
+ return []string{language}
}
-func getLangAndSafe(filename string, content []byte, getLanguageByStrategy Strategy) (lang string, safe bool) {
- languages := getLanguageByStrategy(filename, content)
- if len(languages) == 0 {
- lang = OtherLanguage
- return
+// GetLanguagesByFilename returns a slice of possible languages for the given filename, content and candidates
+// will be ignored. It is comply with the signature to be a Strategy type.
+func GetLanguagesByFilename(filename string, content []byte, candidates []string) []string {
+ return languagesByFilename[filepath.Base(filename)]
+}
+
+// GetLanguagesByShebang returns a slice of possible languages for the given content, filename and candidates
+// will be ignored. It is comply with the signature to be a Strategy type.
+func GetLanguagesByShebang(filename string, content []byte, candidates []string) (languages []string) {
+ interpreter := getInterpreter(content)
+ return languagesByInterpreter[interpreter]
+}
+
+var (
+ shebangExecHack = regexp.MustCompile(`exec (\w+).+\$0.+\$@`)
+ pythonVersion = regexp.MustCompile(`python\d\.\d+`)
+)
+
+func getInterpreter(data []byte) (interpreter string) {
+ line := getFirstLine(data)
+ if !hasShebang(line) {
+ return ""
+ }
+
+ // skip shebang
+ line = bytes.TrimSpace(line[2:])
+
+ splitted := bytes.Fields(line)
+ if bytes.Contains(splitted[0], []byte("env")) {
+ if len(splitted) > 1 {
+ interpreter = string(splitted[1])
+ }
+ } else {
+
+ splittedPath := bytes.Split(splitted[0], []byte{'/'})
+ interpreter = string(splittedPath[len(splittedPath)-1])
+ }
+
+ if interpreter == "sh" {
+ interpreter = lookForMultilineExec(data)
+ }
+
+ if pythonVersion.MatchString(interpreter) {
+ interpreter = interpreter[:strings.Index(interpreter, `.`)]
}
- lang = languages[0]
- safe = len(languages) == 1
return
}
-// GetLanguageByClassifier takes in a content and a list of candidates, and apply the classifier's Classify method to
-// get the most probably language. If classifier is null then DefaultClassfier will be used. If there aren't candidates
-// OtherLanguage is returned.
-func GetLanguageByClassifier(content []byte, candidates map[string]float64, classifier Classifier) string {
- scores := GetLanguagesByClassifier(content, candidates, classifier)
- if len(scores) == 0 {
- return OtherLanguage
+func getFirstLine(data []byte) []byte {
+ buf := bufio.NewScanner(bytes.NewReader(data))
+ buf.Scan()
+ line := buf.Bytes()
+ if err := buf.Err(); err != nil {
+ return nil
}
- return getLangugeHigherScore(scores)
+ return line
}
-func getLangugeHigherScore(scores map[string]float64) string {
- var language string
- higher := -math.MaxFloat64
- for lang, score := range scores {
- if higher < score {
- language = lang
- higher = score
+func hasShebang(line []byte) bool {
+ const shebang = `#!`
+ prefix := []byte(shebang)
+ return bytes.HasPrefix(line, prefix)
+}
+
+func lookForMultilineExec(data []byte) string {
+ const magicNumOfLines = 5
+ interpreter := "sh"
+
+ buf := bufio.NewScanner(bytes.NewReader(data))
+ for i := 0; i < magicNumOfLines && buf.Scan(); i++ {
+ line := buf.Bytes()
+ if shebangExecHack.Match(line) {
+ interpreter = shebangExecHack.FindStringSubmatch(string(line))[1]
+ break
}
}
- return language
+ if err := buf.Err(); err != nil {
+ return interpreter
+ }
+
+ return interpreter
+}
+
+// GetLanguagesByExtension returns a slice of possible languages for the given filename, content and candidates
+// will be ignored. It is comply with the signature to be a Strategy type.
+func GetLanguagesByExtension(filename string, content []byte, candidates []string) []string {
+ if !strings.Contains(filename, ".") {
+ return nil
+ }
+
+ filename = strings.ToLower(filename)
+ dots := getDotIndexes(filename)
+ for _, dot := range dots {
+ ext := filename[dot:]
+ languages, ok := languagesByExtension[ext]
+ if ok {
+ return languages
+ }
+ }
+
+ return nil
+}
+
+func getDotIndexes(filename string) []int {
+ dots := make([]int, 0, 2)
+ for i, letter := range filename {
+ if letter == rune('.') {
+ dots = append(dots, i)
+ }
+ }
+
+ return dots
+}
+
+// GetLanguagesByContent returns a slice of possible languages for the given content, filename and candidates
+// will be ignored. It is comply with the signature to be a Strategy type.
+func GetLanguagesByContent(filename string, content []byte, candidates []string) []string {
+ ext := strings.ToLower(filepath.Ext(filename))
+ fnMatcher, ok := contentMatchers[ext]
+ if !ok {
+ return nil
+ }
+
+ return fnMatcher(content)
+}
+
+// GetLanguagesByClassifier uses DefaultClassifier as a Classifier and returns a sorted slice of possible languages ordered by
+// decreasing language's probability. If there are not candidates it returns nil. It is comply with the signature to be a Strategy type.
+func GetLanguagesByClassifier(filename string, content []byte, candidates []string) (languages []string) {
+ if len(candidates) == 0 {
+ return nil
+ }
+
+ return GetLanguagesBySpecificClassifier(content, candidates, DefaultClassifier)
}
-// GetLanguagesByClassifier returns a map of possible languages as keys and a score as value based on content and candidates. The values can be ordered
-// with the highest value as the most probably language. If classifier is null then DefaultClassfier will be used.
-func GetLanguagesByClassifier(content []byte, candidates map[string]float64, classifier Classifier) map[string]float64 {
- if classifier == nil {
- classifier = DefaultClassifier
+// GetLanguagesBySpecificClassifier returns a slice of possible languages. It takes in a Classifier to be used.
+func GetLanguagesBySpecificClassifier(content []byte, candidates []string, classifier Classifier) (languages []string) {
+ mapCandidates := make(map[string]float64)
+ for _, candidate := range candidates {
+ mapCandidates[candidate]++
}
- return classifier.Classify(content, candidates)
+ return classifier.Classify(content, mapCandidates)
}
// GetLanguageExtensions returns the different extensions being used by the language.
@@ -164,7 +412,7 @@ const (
Prose
)
-// GetLanguageType returns the given language's type.
+// GetLanguageType returns the type of the given language.
func GetLanguageType(language string) (langType Type) {
langType, ok := languagesType[language]
if !ok {
diff --git a/common_test.go b/common_test.go
index 3a0b7790..8ae456cb 100644
--- a/common_test.go
+++ b/common_test.go
@@ -6,7 +6,6 @@ import (
"os"
"path/filepath"
"testing"
- "text/tabwriter"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
@@ -26,6 +25,7 @@ func (s *SimpleLinguistTestSuite) TestGetLanguage() {
filename string
content []byte
expected string
+ safe bool
}{
{name: "TestGetLanguage_1", filename: "foo.py", content: []byte{}, expected: "Python"},
{name: "TestGetLanguage_2", filename: "foo.m", content: []byte(":- module"), expected: "Mercury"},
@@ -38,65 +38,64 @@ func (s *SimpleLinguistTestSuite) TestGetLanguage() {
}
}
-func (s *SimpleLinguistTestSuite) TestGetLanguageByModelineLinguist() {
+func (s *SimpleLinguistTestSuite) TestGetLanguagesByModelineLinguist() {
const (
modelinesDir = ".linguist/test/fixtures/Data/Modelines"
samplesDir = ".linguist/samples"
)
tests := []struct {
- name string
- filename string
- expectedLang string
- expectedSafe bool
+ name string
+ filename string
+ candidates []string
+ expected []string
}{
// Emacs
- {name: "TestGetLanguageByModelineLinguist_1", filename: filepath.Join(modelinesDir, "example_smalltalk.md"), expectedLang: "Smalltalk", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_2", filename: filepath.Join(modelinesDir, "fundamentalEmacs.c"), expectedLang: "Text", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_3", filename: filepath.Join(modelinesDir, "iamphp.inc"), expectedLang: "PHP", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_4", filename: filepath.Join(modelinesDir, "seeplusplusEmacs1"), expectedLang: "C++", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_5", filename: filepath.Join(modelinesDir, "seeplusplusEmacs2"), expectedLang: "C++", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_6", filename: filepath.Join(modelinesDir, "seeplusplusEmacs3"), expectedLang: "C++", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_7", filename: filepath.Join(modelinesDir, "seeplusplusEmacs4"), expectedLang: "C++", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_8", filename: filepath.Join(modelinesDir, "seeplusplusEmacs5"), expectedLang: "C++", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_9", filename: filepath.Join(modelinesDir, "seeplusplusEmacs6"), expectedLang: "C++", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_10", filename: filepath.Join(modelinesDir, "seeplusplusEmacs7"), expectedLang: "C++", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_11", filename: filepath.Join(modelinesDir, "seeplusplusEmacs9"), expectedLang: "C++", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_12", filename: filepath.Join(modelinesDir, "seeplusplusEmacs10"), expectedLang: "C++", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_13", filename: filepath.Join(modelinesDir, "seeplusplusEmacs11"), expectedLang: "C++", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_14", filename: filepath.Join(modelinesDir, "seeplusplusEmacs12"), expectedLang: "C++", expectedSafe: true},
+ {name: "TestGetLanguagesByModelineLinguist_1", filename: filepath.Join(modelinesDir, "example_smalltalk.md"), expected: []string{"Smalltalk"}},
+ {name: "TestGetLanguagesByModelineLinguist_2", filename: filepath.Join(modelinesDir, "fundamentalEmacs.c"), expected: []string{"Text"}},
+ {name: "TestGetLanguagesByModelineLinguist_3", filename: filepath.Join(modelinesDir, "iamphp.inc"), expected: []string{"PHP"}},
+ {name: "TestGetLanguagesByModelineLinguist_4", filename: filepath.Join(modelinesDir, "seeplusplusEmacs1"), expected: []string{"C++"}},
+ {name: "TestGetLanguagesByModelineLinguist_5", filename: filepath.Join(modelinesDir, "seeplusplusEmacs2"), expected: []string{"C++"}},
+ {name: "TestGetLanguagesByModelineLinguist_6", filename: filepath.Join(modelinesDir, "seeplusplusEmacs3"), expected: []string{"C++"}},
+ {name: "TestGetLanguagesByModelineLinguist_7", filename: filepath.Join(modelinesDir, "seeplusplusEmacs4"), expected: []string{"C++"}},
+ {name: "TestGetLanguagesByModelineLinguist_8", filename: filepath.Join(modelinesDir, "seeplusplusEmacs5"), expected: []string{"C++"}},
+ {name: "TestGetLanguagesByModelineLinguist_9", filename: filepath.Join(modelinesDir, "seeplusplusEmacs6"), expected: []string{"C++"}},
+ {name: "TestGetLanguagesByModelineLinguist_10", filename: filepath.Join(modelinesDir, "seeplusplusEmacs7"), expected: []string{"C++"}},
+ {name: "TestGetLanguagesByModelineLinguist_11", filename: filepath.Join(modelinesDir, "seeplusplusEmacs9"), expected: []string{"C++"}},
+ {name: "TestGetLanguagesByModelineLinguist_12", filename: filepath.Join(modelinesDir, "seeplusplusEmacs10"), expected: []string{"C++"}},
+ {name: "TestGetLanguagesByModelineLinguist_13", filename: filepath.Join(modelinesDir, "seeplusplusEmacs11"), expected: []string{"C++"}},
+ {name: "TestGetLanguagesByModelineLinguist_14", filename: filepath.Join(modelinesDir, "seeplusplusEmacs12"), expected: []string{"C++"}},
// Vim
- {name: "TestGetLanguageByModelineLinguist_15", filename: filepath.Join(modelinesDir, "seeplusplus"), expectedLang: "C++", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_16", filename: filepath.Join(modelinesDir, "iamjs.pl"), expectedLang: "JavaScript", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_17", filename: filepath.Join(modelinesDir, "iamjs2.pl"), expectedLang: "JavaScript", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_18", filename: filepath.Join(modelinesDir, "not_perl.pl"), expectedLang: "Prolog", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_19", filename: filepath.Join(modelinesDir, "ruby"), expectedLang: "Ruby", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_20", filename: filepath.Join(modelinesDir, "ruby2"), expectedLang: "Ruby", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_21", filename: filepath.Join(modelinesDir, "ruby3"), expectedLang: "Ruby", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_22", filename: filepath.Join(modelinesDir, "ruby4"), expectedLang: "Ruby", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_23", filename: filepath.Join(modelinesDir, "ruby5"), expectedLang: "Ruby", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_24", filename: filepath.Join(modelinesDir, "ruby6"), expectedLang: "Ruby", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_25", filename: filepath.Join(modelinesDir, "ruby7"), expectedLang: "Ruby", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_26", filename: filepath.Join(modelinesDir, "ruby8"), expectedLang: "Ruby", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_27", filename: filepath.Join(modelinesDir, "ruby9"), expectedLang: "Ruby", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_28", filename: filepath.Join(modelinesDir, "ruby10"), expectedLang: "Ruby", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_29", filename: filepath.Join(modelinesDir, "ruby11"), expectedLang: "Ruby", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_30", filename: filepath.Join(modelinesDir, "ruby12"), expectedLang: "Ruby", expectedSafe: true},
- {name: "TestGetLanguageByModelineLinguist_31", filename: filepath.Join(samplesDir, "C/main.c"), expectedLang: OtherLanguage, expectedSafe: false},
+ {name: "TestGetLanguagesByModelineLinguist_15", filename: filepath.Join(modelinesDir, "seeplusplus"), expected: []string{"C++"}},
+ {name: "TestGetLanguagesByModelineLinguist_16", filename: filepath.Join(modelinesDir, "iamjs.pl"), expected: []string{"JavaScript"}},
+ {name: "TestGetLanguagesByModelineLinguist_17", filename: filepath.Join(modelinesDir, "iamjs2.pl"), expected: []string{"JavaScript"}},
+ {name: "TestGetLanguagesByModelineLinguist_18", filename: filepath.Join(modelinesDir, "not_perl.pl"), expected: []string{"Prolog"}},
+ {name: "TestGetLanguagesByModelineLinguist_19", filename: filepath.Join(modelinesDir, "ruby"), expected: []string{"Ruby"}},
+ {name: "TestGetLanguagesByModelineLinguist_20", filename: filepath.Join(modelinesDir, "ruby2"), expected: []string{"Ruby"}},
+ {name: "TestGetLanguagesByModelineLinguist_21", filename: filepath.Join(modelinesDir, "ruby3"), expected: []string{"Ruby"}},
+ {name: "TestGetLanguagesByModelineLinguist_22", filename: filepath.Join(modelinesDir, "ruby4"), expected: []string{"Ruby"}},
+ {name: "TestGetLanguagesByModelineLinguist_23", filename: filepath.Join(modelinesDir, "ruby5"), expected: []string{"Ruby"}},
+ {name: "TestGetLanguagesByModelineLinguist_24", filename: filepath.Join(modelinesDir, "ruby6"), expected: []string{"Ruby"}},
+ {name: "TestGetLanguagesByModelineLinguist_25", filename: filepath.Join(modelinesDir, "ruby7"), expected: []string{"Ruby"}},
+ {name: "TestGetLanguagesByModelineLinguist_26", filename: filepath.Join(modelinesDir, "ruby8"), expected: []string{"Ruby"}},
+ {name: "TestGetLanguagesByModelineLinguist_27", filename: filepath.Join(modelinesDir, "ruby9"), expected: []string{"Ruby"}},
+ {name: "TestGetLanguagesByModelineLinguist_28", filename: filepath.Join(modelinesDir, "ruby10"), expected: []string{"Ruby"}},
+ {name: "TestGetLanguagesByModelineLinguist_29", filename: filepath.Join(modelinesDir, "ruby11"), expected: []string{"Ruby"}},
+ {name: "TestGetLanguagesByModelineLinguist_30", filename: filepath.Join(modelinesDir, "ruby12"), expected: []string{"Ruby"}},
+ {name: "TestGetLanguagesByModelineLinguist_31", filename: filepath.Join(samplesDir, "C/main.c"), expected: nil},
}
for _, test := range tests {
content, err := ioutil.ReadFile(test.filename)
assert.NoError(s.T(), err)
- lang, safe := GetLanguageByModeline(content)
- assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang))
- assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe))
+ languages := GetLanguagesByModeline(test.filename, content, test.candidates)
+ assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected))
}
}
-func (s *SimpleLinguistTestSuite) TestGetLanguageByModeline() {
+func (s *SimpleLinguistTestSuite) TestGetLanguagesByModeline() {
const (
wrongVim = `# vim: set syntax=ruby ft =python filetype=perl :`
rightVim = `/* vim: set syntax=python ft =python filetype=python */`
@@ -104,48 +103,48 @@ func (s *SimpleLinguistTestSuite) TestGetLanguageByModeline() {
)
tests := []struct {
- name string
- content []byte
- expectedLang string
- expectedSafe bool
+ name string
+ filename string
+ content []byte
+ candidates []string
+ expected []string
}{
- {name: "TestGetLanguageByModeline_1", content: []byte(wrongVim), expectedLang: OtherLanguage, expectedSafe: false},
- {name: "TestGetLanguageByModeline_2", content: []byte(rightVim), expectedLang: "Python", expectedSafe: true},
- {name: "TestGetLanguageByModeline_3", content: []byte(noLangVim), expectedLang: OtherLanguage, expectedSafe: false},
+ {name: "TestGetLanguagesByModeline_1", content: []byte(wrongVim), expected: nil},
+ {name: "TestGetLanguagesByModeline_2", content: []byte(rightVim), expected: []string{"Python"}},
+ {name: "TestGetLanguagesByModeline_3", content: []byte(noLangVim), expected: nil},
}
for _, test := range tests {
- lang, safe := GetLanguageByModeline(test.content)
- assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang))
- assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe))
+ languages := GetLanguagesByModeline(test.filename, test.content, test.candidates)
+ assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected))
}
}
-func (s *SimpleLinguistTestSuite) TestGetLanguageByFilename() {
+func (s *SimpleLinguistTestSuite) TestGetLanguagesByFilename() {
tests := []struct {
- name string
- filename string
- expectedLang string
- expectedSafe bool
+ name string
+ filename string
+ content []byte
+ candidates []string
+ expected []string
}{
- {name: "TestGetLanguageByFilename_1", filename: "unknown.interpreter", expectedLang: OtherLanguage, expectedSafe: false},
- {name: "TestGetLanguageByFilename_2", filename: ".bashrc", expectedLang: "Shell", expectedSafe: true},
- {name: "TestGetLanguageByFilename_3", filename: "Dockerfile", expectedLang: "Dockerfile", expectedSafe: true},
- {name: "TestGetLanguageByFilename_4", filename: "Makefile.frag", expectedLang: "Makefile", expectedSafe: true},
- {name: "TestGetLanguageByFilename_5", filename: "makefile", expectedLang: "Makefile", expectedSafe: true},
- {name: "TestGetLanguageByFilename_6", filename: "Vagrantfile", expectedLang: "Ruby", expectedSafe: true},
- {name: "TestGetLanguageByFilename_7", filename: "_vimrc", expectedLang: "Vim script", expectedSafe: true},
- {name: "TestGetLanguageByFilename_8", filename: "pom.xml", expectedLang: "Maven POM", expectedSafe: true},
+ {name: "TestGetLanguagesByFilename_1", filename: "unknown.interpreter", expected: nil},
+ {name: "TestGetLanguagesByFilename_2", filename: ".bashrc", expected: []string{"Shell"}},
+ {name: "TestGetLanguagesByFilename_3", filename: "Dockerfile", expected: []string{"Dockerfile"}},
+ {name: "TestGetLanguagesByFilename_4", filename: "Makefile.frag", expected: []string{"Makefile"}},
+ {name: "TestGetLanguagesByFilename_5", filename: "makefile", expected: []string{"Makefile"}},
+ {name: "TestGetLanguagesByFilename_6", filename: "Vagrantfile", expected: []string{"Ruby"}},
+ {name: "TestGetLanguagesByFilename_7", filename: "_vimrc", expected: []string{"Vim script"}},
+ {name: "TestGetLanguagesByFilename_8", filename: "pom.xml", expected: []string{"Maven POM"}},
}
for _, test := range tests {
- lang, safe := GetLanguageByFilename(test.filename)
- assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang))
- assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe))
+ languages := GetLanguagesByFilename(test.filename, test.content, test.candidates)
+ assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected))
}
}
-func (s *SimpleLinguistTestSuite) TestGetLanguageByShebang() {
+func (s *SimpleLinguistTestSuite) TestGetLanguagesByShebang() {
const (
multilineExecHack = `#!/bin/sh
# Next line is comment in Tcl, but not in sh... \
@@ -162,125 +161,112 @@ println("The shell script says ",vm.arglist.concat(" "));`
)
tests := []struct {
- name string
- content []byte
- expectedLang string
- expectedSafe bool
+ name string
+ filename string
+ content []byte
+ candidates []string
+ expected []string
}{
- {name: "TestGetLanguageByShebang_1", content: []byte(`#!/unknown/interpreter`), expectedLang: OtherLanguage, expectedSafe: false},
- {name: "TestGetLanguageByShebang_2", content: []byte(`no shebang`), expectedLang: OtherLanguage, expectedSafe: false},
- {name: "TestGetLanguageByShebang_3", content: []byte(`#!/usr/bin/env`), expectedLang: OtherLanguage, expectedSafe: false},
- {name: "TestGetLanguageByShebang_4", content: []byte(`#!/usr/bin/python -tt`), expectedLang: "Python", expectedSafe: true},
- {name: "TestGetLanguageByShebang_5", content: []byte(`#!/usr/bin/env python2.6`), expectedLang: "Python", expectedSafe: true},
- {name: "TestGetLanguageByShebang_6", content: []byte(`#!/usr/bin/env perl`), expectedLang: "Perl", expectedSafe: true},
- {name: "TestGetLanguageByShebang_7", content: []byte(`#! /bin/sh`), expectedLang: "Shell", expectedSafe: true},
- {name: "TestGetLanguageByShebang_8", content: []byte(`#!bash`), expectedLang: "Shell", expectedSafe: true},
- {name: "TestGetLanguageByShebang_9", content: []byte(multilineExecHack), expectedLang: "Tcl", expectedSafe: true},
- {name: "TestGetLanguageByShebang_10", content: []byte(multilineNoExecHack), expectedLang: "Shell", expectedSafe: true},
+ {name: "TestGetLanguagesByShebang_1", content: []byte(`#!/unknown/interpreter`), expected: nil},
+ {name: "TestGetLanguagesByShebang_2", content: []byte(`no shebang`), expected: nil},
+ {name: "TestGetLanguagesByShebang_3", content: []byte(`#!/usr/bin/env`), expected: nil},
+ {name: "TestGetLanguagesByShebang_4", content: []byte(`#!/usr/bin/python -tt`), expected: []string{"Python"}},
+ {name: "TestGetLanguagesByShebang_5", content: []byte(`#!/usr/bin/env python2.6`), expected: []string{"Python"}},
+ {name: "TestGetLanguagesByShebang_6", content: []byte(`#!/usr/bin/env perl`), expected: []string{"Perl"}},
+ {name: "TestGetLanguagesByShebang_7", content: []byte(`#! /bin/sh`), expected: []string{"Shell"}},
+ {name: "TestGetLanguagesByShebang_8", content: []byte(`#!bash`), expected: []string{"Shell"}},
+ {name: "TestGetLanguagesByShebang_9", content: []byte(multilineExecHack), expected: []string{"Tcl"}},
+ {name: "TestGetLanguagesByShebang_10", content: []byte(multilineNoExecHack), expected: []string{"Shell"}},
}
for _, test := range tests {
- lang, safe := GetLanguageByShebang(test.content)
- assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang))
- assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe))
+ languages := GetLanguagesByShebang(test.filename, test.content, test.candidates)
+ assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected))
}
}
-func (s *SimpleLinguistTestSuite) TestGetLanguageByExtension() {
+func (s *SimpleLinguistTestSuite) TestGetLanguagesByExtension() {
tests := []struct {
- name string
- filename string
- expectedLang string
- expectedSafe bool
+ name string
+ filename string
+ content []byte
+ candidates []string
+ expected []string
}{
- {name: "TestGetLanguageByExtension_1", filename: "foo.foo", expectedLang: OtherLanguage, expectedSafe: false},
- {name: "TestGetLanguageByExtension_2", filename: "foo.go", expectedLang: "Go", expectedSafe: true},
- {name: "TestGetLanguageByExtension_3", filename: "foo.go.php", expectedLang: "Hack", expectedSafe: false},
+ {name: "TestGetLanguagesByExtension_1", filename: "foo.foo", expected: nil},
+ {name: "TestGetLanguagesByExtension_2", filename: "foo.go", expected: []string{"Go"}},
+ {name: "TestGetLanguagesByExtension_3", filename: "foo.go.php", expected: []string{"Hack", "PHP"}},
}
for _, test := range tests {
- lang, safe := GetLanguageByExtension(test.filename)
- assert.Equal(s.T(), test.expectedLang, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expectedLang))
- assert.Equal(s.T(), test.expectedSafe, safe, fmt.Sprintf("%v: safe = %v, expected: %v", test.name, safe, test.expectedSafe))
+ languages := GetLanguagesByExtension(test.filename, test.content, test.candidates)
+ assert.Equal(s.T(), test.expected, languages, fmt.Sprintf("%v: languages = %v, expected: %v", test.name, languages, test.expected))
}
}
-func (s *SimpleLinguistTestSuite) TestGetLanguageByContentLinguistCorpus() {
- var total, failed, ok, other, unsafe int
-
- w := new(tabwriter.Writer)
- w.Init(os.Stdout, 0, 8, 0, '\t', 0)
-
- filepath.Walk(".linguist/samples", func(path string, f os.FileInfo, err error) error {
- if f.IsDir() {
- if f.Name() == "filenames" {
- return filepath.SkipDir
- }
-
- return nil
- }
-
- expected := filepath.Base(filepath.Dir(path))
- filename := filepath.Base(path)
- extension := filepath.Ext(path)
- content, _ := ioutil.ReadFile(path)
-
- if extension == "" {
- return nil
- }
+func (s *SimpleLinguistTestSuite) TestGetLanguagesByClassifier() {
+ const samples = `.linguist/samples/`
+ test := []struct {
+ name string
+ filename string
+ candidates []string
+ expected string
+ }{
+ {name: "TestGetLanguagesByClassifier_1", filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"python", "ruby", "c", "c++"}, expected: "C"},
+ {name: "TestGetLanguagesByClassifier_2", filename: filepath.Join(samples, "C/blob.c"), candidates: nil, expected: OtherLanguage},
+ {name: "TestGetLanguagesByClassifier_3", filename: filepath.Join(samples, "C/main.c"), candidates: []string{}, expected: OtherLanguage},
+ {name: "TestGetLanguagesByClassifier_4", filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"python", "ruby", "c++"}, expected: "C++"},
+ {name: "TestGetLanguagesByClassifier_5", filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"ruby"}, expected: "Ruby"},
+ {name: "TestGetLanguagesByClassifier_6", filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: []string{"python", "ruby", "c", "c++"}, expected: "Python"},
+ }
- total++
- obtained, safe := GetLanguageByContent(filename, content)
- if obtained == OtherLanguage {
- other++
- }
+ for _, test := range test {
+ content, err := ioutil.ReadFile(test.filename)
+ assert.NoError(s.T(), err)
- var status string
- if expected == obtained {
- status = "ok"
- ok++
+ languages := GetLanguagesByClassifier(test.filename, content, test.candidates)
+ var language string
+ if len(languages) == 0 {
+ language = OtherLanguage
} else {
- status = "failed"
- failed++
- if !safe {
- unsafe++
- }
+ language = languages[0]
}
- fmt.Fprintf(w, "%s\t%s\t%s\t%v\t%s\n", filename, expected, obtained, safe, status)
-
- return nil
- })
-
- fmt.Fprintln(w)
- w.Flush()
-
- fmt.Printf("total files: %d, ok: %d, failed: %d, unsafe: %d, other: %d\n", total, ok, failed, unsafe, other)
+ assert.Equal(s.T(), test.expected, language, fmt.Sprintf("%v: language = %v, expected: %v", test.name, language, test.expected))
+ }
}
-func (s *SimpleLinguistTestSuite) TestGetLanguageByClassifier() {
+func (s *SimpleLinguistTestSuite) TestGetLanguagesBySpecificClassifier() {
const samples = `.linguist/samples/`
test := []struct {
name string
filename string
- candidates map[string]float64
+ candidates []string
+ classifier Classifier
expected string
}{
- {name: "TestGetLanguageByClassifier_1", filename: filepath.Join(samples, "C/blob.c"), candidates: map[string]float64{"python": 1.00, "ruby": 1.00, "c": 1.00, "c++": 1.00}, expected: "C"},
- {name: "TestGetLanguageByClassifier_2", filename: filepath.Join(samples, "C/blob.c"), candidates: nil, expected: "C"},
- {name: "TestGetLanguageByClassifier_3", filename: filepath.Join(samples, "C/main.c"), candidates: nil, expected: "C"},
- {name: "TestGetLanguageByClassifier_4", filename: filepath.Join(samples, "C/blob.c"), candidates: map[string]float64{"python": 1.00, "ruby": 1.00, "c++": 1.00}, expected: "C++"},
- {name: "TestGetLanguageByClassifier_5", filename: filepath.Join(samples, "C/blob.c"), candidates: map[string]float64{"ruby": 1.00}, expected: "Ruby"},
- {name: "TestGetLanguageByClassifier_6", filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: map[string]float64{"python": 1.00, "ruby": 1.00, "c": 1.00, "c++": 1.00}, expected: "Python"},
- {name: "TestGetLanguageByClassifier_7", filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: nil, expected: "Python"},
+ {name: "TestGetLanguagesByClassifier_1", filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"python", "ruby", "c", "c++"}, classifier: DefaultClassifier, expected: "C"},
+ {name: "TestGetLanguagesByClassifier_2", filename: filepath.Join(samples, "C/blob.c"), candidates: nil, classifier: DefaultClassifier, expected: "C"},
+ {name: "TestGetLanguagesByClassifier_3", filename: filepath.Join(samples, "C/main.c"), candidates: []string{}, classifier: DefaultClassifier, expected: "C"},
+ {name: "TestGetLanguagesByClassifier_4", filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"python", "ruby", "c++"}, classifier: DefaultClassifier, expected: "C++"},
+ {name: "TestGetLanguagesByClassifier_5", filename: filepath.Join(samples, "C/blob.c"), candidates: []string{"ruby"}, classifier: DefaultClassifier, expected: "Ruby"},
+ {name: "TestGetLanguagesByClassifier_6", filename: filepath.Join(samples, "Python/django-models-base.py"), candidates: []string{"python", "ruby", "c", "c++"}, classifier: DefaultClassifier, expected: "Python"},
+ {name: "TestGetLanguagesByClassifier_6", filename: os.DevNull, candidates: nil, classifier: DefaultClassifier, expected: OtherLanguage},
}
for _, test := range test {
content, err := ioutil.ReadFile(test.filename)
assert.NoError(s.T(), err)
- lang := GetLanguageByClassifier(content, test.candidates, nil)
- assert.Equal(s.T(), test.expected, lang, fmt.Sprintf("%v: lang = %v, expected: %v", test.name, lang, test.expected))
+ languages := GetLanguagesBySpecificClassifier(content, test.candidates, test.classifier)
+ var language string
+ if len(languages) == 0 {
+ language = OtherLanguage
+ } else {
+ language = languages[0]
+ }
+
+ assert.Equal(s.T(), test.expected, language, fmt.Sprintf("%v: language = %v, expected: %v", test.name, language, test.expected))
}
}
@@ -349,3 +335,56 @@ func (s *SimpleLinguistTestSuite) TestGetLanguageByAlias() {
assert.Equal(s.T(), test.expectedOk, ok, fmt.Sprintf("%v: ok = %v, expected: %v", test.name, ok, test.expectedOk))
}
}
+
+func (s *SimpleLinguistTestSuite) TestLinguistCorpus() {
+ const (
+ samplesDir = ".linguist/samples"
+ filenamesDir = "filenames"
+ )
+
+ var cornerCases = map[string]bool{
+ "hello.ms": true,
+ }
+
+ var total, failed, ok, other int
+ var expected string
+ filepath.Walk(samplesDir, func(path string, f os.FileInfo, err error) error {
+ if f.IsDir() {
+ if f.Name() != filenamesDir {
+ expected = f.Name()
+ }
+
+ return nil
+ }
+
+ filename := filepath.Base(path)
+ content, _ := ioutil.ReadFile(path)
+
+ total++
+ obtained := GetLanguage(filename, content)
+ if obtained == OtherLanguage {
+ obtained = "Other"
+ other++
+ }
+
+ var status string
+ if expected == obtained {
+ status = "ok"
+ ok++
+ } else {
+ status = "failed"
+ failed++
+
+ }
+
+ if _, ok := cornerCases[filename]; ok {
+ fmt.Printf("\t\t[considered corner case] %s\texpected: %s\tobtained: %s\tstatus: %s\n", filename, expected, obtained, status)
+ } else {
+ assert.Equal(s.T(), expected, obtained, fmt.Sprintf("%s\texpected: %s\tobtained: %s\tstatus: %s\n", filename, expected, obtained, status))
+ }
+
+ return nil
+ })
+
+ fmt.Printf("\t\ttotal files: %d, ok: %d, failed: %d, other: %d\n", total, ok, failed, other)
+}
diff --git a/content.go b/content.go
index 19745283..a7eaa969 100644
--- a/content.go
+++ b/content.go
@@ -507,7 +507,7 @@ var (
m_Matlab_Matcher_0 = regexp.MustCompile(`(?m)^\s*%`)
m_Limbo_Matcher_0 = regexp.MustCompile(`(?m)^\w+\s*:\s*module\s*{`)
md_Markdown_Matcher_0 = regexp.MustCompile(`(?mi)(^[-a-z0-9=#!\*\[|>])|<\/`)
- md_Markdown_Matcher_1 = regexp.MustCompile(`(?m)^$`)
+ md_Markdown_Matcher_1 = regexp.MustCompile(`^$`)
md_GCCMachineDescription_Matcher_0 = regexp.MustCompile(`(?m)^(;;|\(define_)`)
ml_OCaml_Matcher_0 = regexp.MustCompile(`(?m)(^\s*module)|let rec |match\s+(\S+\s)+with`)
ml_StandardML_Matcher_0 = regexp.MustCompile(`(?m)=> |case\s+(\S+\s)+of`)
diff --git a/extension.go b/extension.go
index 455ec22e..247ce12e 100644
--- a/extension.go
+++ b/extension.go
@@ -28,9 +28,6 @@ var languagesByExtension = map[string][]string{
".8xp": {"TI Program"},
".8xp.txt": {"TI Program"},
".9": {"Roff"},
- ".E": {"E"},
- ".JSON-tmLanguage": {"JSON"},
- ".ML": {"Standard ML"},
"._coffee": {"CoffeeScript"},
"._js": {"JavaScript"},
"._ls": {"LiveScript"},
@@ -228,7 +225,7 @@ var languagesByExtension = map[string][]string{
".dyalog": {"APL"},
".dyl": {"Dylan"},
".dylan": {"Dylan"},
- ".e": {"Eiffel"},
+ ".e": {"E", "Eiffel"},
".eam.fs": {"Formatted"},
".ebnf": {"EBNF"},
".ebuild": {"Gentoo Ebuild"},
@@ -415,6 +412,7 @@ var languagesByExtension = map[string][]string{
".jsfl": {"JavaScript"},
".jsm": {"JavaScript"},
".json": {"JSON"},
+ ".json-tmlanguage": {"JSON"},
".json5": {"JSON5"},
".jsonld": {"JSONLD"},
".jsp": {"Java Server Pages"},
@@ -508,7 +506,7 @@ var languagesByExtension = map[string][]string{
".mkii": {"TeX"},
".mkiv": {"TeX"},
".mkvi": {"TeX"},
- ".ml": {"OCaml"},
+ ".ml": {"OCaml", "Standard ML"},
".ml4": {"OCaml"},
".mli": {"OCaml"},
".mll": {"OCaml"},
@@ -807,11 +805,11 @@ var languagesByExtension = map[string][]string{
".ss": {"Scheme"},
".ssjs": {"JavaScript"},
".st": {"HTML", "Smalltalk"},
- ".stTheme": {"XML"},
".stan": {"Stan"},
".sthlp": {"Stata"},
".ston": {"STON"},
".storyboard": {"XML"},
+ ".sttheme": {"XML"},
".sty": {"TeX"},
".styl": {"Stylus"},
".sublime-build": {"Sublime Text Config"},
@@ -853,13 +851,13 @@ var languagesByExtension = map[string][]string{
".tl": {"Type Language"},
".tla": {"TLA"},
".tm": {"Tcl"},
- ".tmCommand": {"XML"},
- ".tmLanguage": {"XML"},
- ".tmPreferences": {"XML"},
- ".tmSnippet": {"XML"},
- ".tmTheme": {"XML"},
".tmac": {"Roff"},
+ ".tmcommand": {"XML"},
".tml": {"XML"},
+ ".tmlanguage": {"XML"},
+ ".tmpreferences": {"XML"},
+ ".tmsnippet": {"XML"},
+ ".tmtheme": {"XML"},
".tmux": {"Shell"},
".toc": {"TeX", "World of Warcraft Addon Data"},
".toml": {"TOML"},
@@ -1099,7 +1097,7 @@ var extensionsByLanguage = map[string][]string{
"Dockerfile": {".dockerfile"},
"Dogescript": {".djs"},
"Dylan": {".dylan", ".dyl", ".intr", ".lid"},
- "E": {".E"},
+ "E": {".e"},
"EBNF": {".ebnf"},
"ECL": {".ecl", ".eclxml"},
"ECLiPSe": {".ecl"},
@@ -1182,7 +1180,7 @@ var extensionsByLanguage = map[string][]string{
"Isabelle": {".thy"},
"J": {".ijs"},
"JFlex": {".flex", ".jflex"},
- "JSON": {".json", ".geojson", ".JSON-tmLanguage", ".topojson"},
+ "JSON": {".json", ".geojson", ".json-tmlanguage", ".topojson"},
"JSON5": {".json5"},
"JSONLD": {".jsonld"},
"JSONiq": {".jq"},
@@ -1379,7 +1377,7 @@ var extensionsByLanguage = map[string][]string{
"Spline Font Database": {".sfd"},
"Squirrel": {".nut"},
"Stan": {".stan"},
- "Standard ML": {".ML", ".fun", ".sig", ".sml"},
+ "Standard ML": {".ml", ".fun", ".sig", ".sml"},
"Stata": {".do", ".ado", ".doh", ".ihlp", ".mata", ".matah", ".sthlp"},
"Stylus": {".styl"},
"SubRip Text": {".srt"},
@@ -1426,7 +1424,7 @@ var extensionsByLanguage = map[string][]string{
"World of Warcraft Addon Data": {".toc"},
"X10": {".x10"},
"XC": {".xc"},
- "XML": {".xml", ".adml", ".admx", ".ant", ".axml", ".builds", ".ccxml", ".clixml", ".cproject", ".csl", ".csproj", ".ct", ".dita", ".ditamap", ".ditaval", ".dll.config", ".dotsettings", ".filters", ".fsproj", ".fxml", ".glade", ".gml", ".grxml", ".iml", ".ivy", ".jelly", ".jsproj", ".kml", ".launch", ".mdpolicy", ".mjml", ".mm", ".mod", ".mxml", ".nproj", ".nuspec", ".odd", ".osm", ".pkgproj", ".plist", ".pluginspec", ".props", ".ps1xml", ".psc1", ".pt", ".rdf", ".resx", ".rss", ".sch", ".scxml", ".sfproj", ".srdf", ".storyboard", ".stTheme", ".sublime-snippet", ".targets", ".tmCommand", ".tml", ".tmLanguage", ".tmPreferences", ".tmSnippet", ".tmTheme", ".ts", ".tsx", ".ui", ".urdf", ".ux", ".vbproj", ".vcxproj", ".vsixmanifest", ".vssettings", ".vstemplate", ".vxml", ".wixproj", ".wsdl", ".wsf", ".wxi", ".wxl", ".wxs", ".x3d", ".xacro", ".xaml", ".xib", ".xlf", ".xliff", ".xmi", ".xml.dist", ".xproj", ".xsd", ".xul", ".zcml"},
+ "XML": {".xml", ".adml", ".admx", ".ant", ".axml", ".builds", ".ccxml", ".clixml", ".cproject", ".csl", ".csproj", ".ct", ".dita", ".ditamap", ".ditaval", ".dll.config", ".dotsettings", ".filters", ".fsproj", ".fxml", ".glade", ".gml", ".grxml", ".iml", ".ivy", ".jelly", ".jsproj", ".kml", ".launch", ".mdpolicy", ".mjml", ".mm", ".mod", ".mxml", ".nproj", ".nuspec", ".odd", ".osm", ".pkgproj", ".plist", ".pluginspec", ".props", ".ps1xml", ".psc1", ".pt", ".rdf", ".resx", ".rss", ".sch", ".scxml", ".sfproj", ".srdf", ".storyboard", ".sttheme", ".sublime-snippet", ".targets", ".tmcommand", ".tml", ".tmlanguage", ".tmpreferences", ".tmsnippet", ".tmtheme", ".ts", ".tsx", ".ui", ".urdf", ".ux", ".vbproj", ".vcxproj", ".vsixmanifest", ".vssettings", ".vstemplate", ".vxml", ".wixproj", ".wsdl", ".wsf", ".wxi", ".wxl", ".wxs", ".x3d", ".xacro", ".xaml", ".xib", ".xlf", ".xliff", ".xmi", ".xml.dist", ".xproj", ".xsd", ".xul", ".zcml"},
"XPages": {".xsp-config", ".xsp.metadata"},
"XProc": {".xpl", ".xproc"},
"XQuery": {".xquery", ".xq", ".xql", ".xqm", ".xqy"},
diff --git a/filename.go b/filename.go
index ac889128..b4f19af8 100644
--- a/filename.go
+++ b/filename.go
@@ -16,22 +16,39 @@ var languagesByFilename = map[string][]string{
".bashrc": {"Shell"},
".clang-format": {"YAML"},
".classpath": {"XML"},
+ ".cproject": {"XML"},
+ ".cshrc": {"Shell"},
+ ".editorconfig": {"INI"},
".emacs": {"Emacs Lisp"},
".emacs.desktop": {"Emacs Lisp"},
".factor-boot-rc": {"Factor"},
".factor-rc": {"Factor"},
".gclient": {"Python"},
+ ".gemrc": {"YAML"},
+ ".gitconfig": {"INI"},
+ ".gn": {"GN"},
".gnus": {"Emacs Lisp"},
+ ".gvimrc": {"Vim script"},
+ ".htaccess": {"ApacheConf"},
".irbrc": {"Ruby"},
".jshintrc": {"JSON"},
+ ".login": {"Shell"},
".nvimrc": {"Vim script"},
+ ".php": {"PHP"},
".php_cs": {"PHP"},
".php_cs.dist": {"PHP"},
+ ".profile": {"Shell"},
".project": {"XML"},
".pryrc": {"Ruby"},
".spacemacs": {"Emacs Lisp"},
".vimrc": {"Vim script"},
".viper": {"Emacs Lisp"},
+ ".zlogin": {"Shell"},
+ ".zlogout": {"Shell"},
+ ".zprofile": {"Shell"},
+ ".zshenv": {"Shell"},
+ ".zshrc": {"Shell"},
+ "9fs": {"Shell"},
"APKBUILD": {"Alpine Abuild"},
"App.config": {"XML"},
"Appraisals": {"Ruby"},
@@ -43,8 +60,10 @@ var languagesByFilename = map[string][]string{
"Buildfile": {"Ruby"},
"CMakeLists.txt": {"CMake"},
"COPYING": {"Text"},
+ "COPYING.regex": {"Text"},
"COPYRIGHT.regex": {"Text"},
"Cakefile": {"CoffeeScript"},
+ "Capfile": {"Ruby"},
"Cask": {"Emacs Lisp"},
"Dangerfile": {"Ruby"},
"Deliverfile": {"Ruby"},
@@ -105,21 +124,31 @@ var languagesByFilename = map[string][]string{
"_emacs": {"Emacs Lisp"},
"_vimrc": {"Vim script"},
"abbrev_defs": {"Emacs Lisp"},
+ "ack": {"Perl"},
"ant.xml": {"Ant Build System"},
+ "apache2.conf": {"ApacheConf"},
+ "bash_logout": {"Shell"},
+ "bash_profile": {"Shell"},
+ "bashrc": {"Shell"},
"build.xml": {"Ant Build System"},
"buildfile": {"Ruby"},
"click.me": {"Text"},
"composer.lock": {"JSON"},
"configure.ac": {"M4Sugar"},
+ "cshrc": {"Shell"},
"delete.me": {"Text"},
"descrip.mmk": {"Module Management System"},
"descrip.mms": {"Module Management System"},
+ "expr-dist": {"R"},
"gradlew": {"Shell"},
"gvimrc": {"Vim script"},
+ "httpd.conf": {"ApacheConf"},
"keep.me": {"Text"},
"ld.script": {"Linker Script"},
+ "login": {"Shell"},
"makefile": {"Makefile"},
"makefile.sco": {"Makefile"},
+ "man": {"Shell"},
"mcmod.info": {"JSON"},
"meson.build": {"Meson"},
"meson_options.txt": {"Meson"},
@@ -129,15 +158,25 @@ var languagesByFilename = map[string][]string{
"mmt": {"Roff"},
"nginx.conf": {"Nginx"},
"nvimrc": {"Vim script"},
+ "owh": {"Tcl"},
"packages.config": {"XML"},
"pom.xml": {"Maven POM"},
+ "profile": {"Shell"},
"read.me": {"Text"},
+ "readme.1st": {"Text"},
"rebar.config": {"Erlang"},
"rebar.config.lock": {"Erlang"},
"rebar.lock": {"Erlang"},
"riemann.config": {"Clojure"},
+ "script": {"C"},
+ "starfield": {"Tcl"},
"test.me": {"Text"},
"vimrc": {"Vim script"},
"wscript": {"Python"},
"xcompose": {"XCompose"},
+ "zlogin": {"Shell"},
+ "zlogout": {"Shell"},
+ "zprofile": {"Shell"},
+ "zshenv": {"Shell"},
+ "zshrc": {"Shell"},
}
diff --git a/frequencies.go b/frequencies.go
index dd9601a7..c9dccb16 100644
--- a/frequencies.go
+++ b/frequencies.go
@@ -386,557 +386,666 @@ var DefaultClassifier Classifier = &classifier{
},
tokensLogProbabilities: map[string]map[string]float64{
"1C Enterprise": map[string]float64{
- "&": -6.672223,
- "(": -4.054827,
- ")": -4.054827,
- "*": -9.667955,
- "+": -6.202219,
- ",": -5.082988,
- "-": -9.667955,
- ".": -3.954222,
- "/": -9.667955,
- ";": -3.874941,
- "<": -6.777583,
- "=": -4.474998,
- ">": -6.723516,
- "?": -8.974808,
- "DOM": -8.974808,
- "DOM.": -9.667955,
- "HTML": -8.974808,
- "HTML.": -8.974808,
- "Null": -9.667955,
- "XML": -7.270060,
- "XML.": -7.722045,
- "_": -7.470730,
- "А": -8.281661,
- "Б": -8.058517,
- "В": -4.698142,
- "Г": -8.281661,
- "Д": -4.698142,
- "Е": -5.114078,
- "Ж": -9.667955,
- "З": -4.586551,
- "И": -5.311246,
- "К": -4.311369,
- "Л": -6.233968,
- "М": -6.266758,
- "Н": -4.586551,
- "О": -4.618099,
- "П": -3.984375,
- "Р": -5.839314,
- "С": -4.431513,
- "Т": -4.400097,
- "У": -6.004393,
- "Ф": -5.979076,
- "Х": -8.974808,
- "Ц": -5.607512,
- "Ч": -6.576913,
- "Ш": -6.623433,
- "Э": -5.796754,
- "а": -2.500146,
- "б": -4.274328,
- "в": -3.881058,
- "г": -4.808143,
- "д": -4.036743,
- "е": -2.538658,
- "ж": -5.146166,
- "з": -4.839641,
- "и": -2.864450,
- "й": -4.684348,
- "к": -3.479691,
- "л": -3.220649,
- "м": -4.076968,
- "н": -2.950150,
- "о": -2.582054,
- "п": -4.301979,
- "р": -3.197156,
- "с": -3.578910,
- "т": -2.868899,
- "у": -4.029600,
- "ф": -5.817807,
- "х": -6.202219,
- "ц": -4.306663,
- "ч": -4.657320,
- "ш": -7.470730,
- "щ": -5.524820,
- "ъ": -5.979076,
- "ы": -3.793024,
- "ь": -4.043938,
- "ю": -6.112607,
- "я": -4.562010,
- "\ufeff": -8.974808,
+ "#": -8.978471,
+ "&": -6.675886,
+ "(": -4.065816,
+ ")": -4.065816,
+ "*": -9.671619,
+ "+": -6.205883,
+ ",": -5.086651,
+ "-": -8.978471,
+ ".": -3.957886,
+ "/": -9.671619,
+ "//": -8.573006,
+ "//////////////////////////////////////////////////////////////////////////////": -8.978471,
+ ";": -3.878605,
+ "<": -6.781247,
+ "=": -4.478662,
+ ">": -6.727180,
+ "?": -8.978471,
+ "DOM": -8.978471,
+ "DOM.": -9.671619,
+ "HTML": -8.978471,
+ "HTML.": -8.978471,
+ "Null": -9.671619,
+ "XML": -7.273723,
+ "XML.": -7.725708,
+ "_": -7.474394,
+ "А": -8.285324,
+ "Б": -8.062181,
+ "В": -4.701805,
+ "Г": -8.285324,
+ "Д": -4.722859,
+ "Е": -5.107270,
+ "Ж": -9.671619,
+ "З": -4.584022,
+ "И": -5.289592,
+ "К": -4.310326,
+ "Л": -6.237631,
+ "М": -6.270421,
+ "Н": -4.590214,
+ "О": -4.641181,
+ "П": -4.001738,
+ "Р": -5.842977,
+ "С": -4.435177,
+ "Т": -4.403760,
+ "У": -6.060701,
+ "Ф": -5.982739,
+ "Х": -8.978471,
+ "Ц": -5.611176,
+ "Ч": -6.580576,
+ "Ш": -6.627096,
+ "Э": -5.821471,
+ "а": -2.506125,
+ "б": -4.291721,
+ "в": -3.860478,
+ "г": -4.804084,
+ "д": -4.036829,
+ "е": -2.541520,
+ "ж": -5.139019,
+ "з": -4.827431,
+ "и": -2.859273,
+ "й": -4.688012,
+ "к": -3.479256,
+ "л": -3.214849,
+ "м": -4.073197,
+ "н": -2.957448,
+ "о": -2.573243,
+ "п": -4.273456,
+ "р": -3.199272,
+ "с": -3.569060,
+ "т": -2.863684,
+ "у": -4.029711,
+ "ф": -5.821471,
+ "х": -6.237631,
+ "ц": -4.319760,
+ "ч": -4.654339,
+ "ш": -7.369033,
+ "щ": -5.528484,
+ "ъ": -5.982739,
+ "ы": -3.805150,
+ "ь": -4.029711,
+ "ю": -6.116270,
+ "я": -4.565673,
+ "\ufeff": -8.978471,
},
"ABAP": map[string]float64{
- "!": -3.263996,
- "(": -4.442651,
- ")": -4.442651,
- "*": -2.427748,
- "+": -4.219508,
- "-": -4.219508,
- ".": -3.631721,
- "<-()]>": -5.135798,
- "": -5.135798,
- "": -4.442651,
- "": -4.730333,
- "=": -3.526361,
- ">": -4.442651,
- "CLASS": -5.135798,
- "CL_CSV_PARSER": -4.219508,
- "CONSTRUCTOR": -5.828946,
- "CSV": -5.828946,
- "CX": -5.828946,
- "CX_CSV_PARSE_ERROR": -5.828946,
- "DEFINITION": -5.135798,
- "Get": -5.828946,
- "IMPLEMENTATION": -5.828946,
- "Instance": -5.135798,
- "Method": -5.135798,
- "Parse": -5.828946,
- "Private": -5.828946,
- "Public": -5.828946,
- "RETURNING": -5.828946,
- "SIGNATURE": -5.135798,
- "STRINGTAB": -4.730333,
- "Space": -5.135798,
- "TYPE": -4.730333,
- "This": -5.828946,
- "[": -4.219508,
- "_LINES": -5.828946,
- "_csvstring": -5.135798,
- "_delegate": -5.828946,
- "_lines": -5.828946,
- "_lines.": -5.828946,
- "_parse_line": -5.135798,
- "_separator": -5.828946,
- "_skip_first_line": -5.828946,
- "_textindicator": -5.828946,
- "`": -4.442651,
- "abap": -5.828946,
- "abap_bool": -5.135798,
- "abap_true.": -5.135798,
- "an": -5.828946,
- "append": -5.135798,
- "assigning": -5.828946,
- "at": -5.135798,
- "c": -5.135798,
- "char": -5.135798,
- "cl_abap_char_utilities": -5.828946,
- "cl_csv_parser": -5.135798,
- "cl_object": -5.828946,
- "class": -5.135798,
- "clear": -5.828946,
- "concatenate": -4.442651,
- "constants": -5.828946,
- "constructor": -5.135798,
- "constructor.": -5.828946,
- "cr_lf": -5.828946,
- "create": -5.828946,
- "csv": -5.828946,
- "csvstring": -5.828946,
- "csvstring.": -5.828946,
- "csvvalue": -4.037186,
- "csvvalue.": -4.219508,
- "csvvalues.": -5.135798,
- "cx_csv_parse_error": -5.135798,
- "data": -4.730333,
- "definition": -5.828946,
- "delegate": -5.828946,
- "delegate.": -5.828946,
- "do": -4.730333,
- "e": -5.828946,
- "else.": -4.442651,
- "endclass.": -5.828946,
- "endif.": -4.037186,
- "endmethod.": -5.135798,
- "endwhile.": -5.135798,
- "error": -5.828946,
- "exception": -5.828946,
- "exporting": -5.828946,
- "field": -5.828946,
- "files": -4.730333,
- "final": -5.828946,
- "formatting": -5.828946,
- "from": -5.828946,
- "here": -4.730333,
- "if_csv_parser_delegate": -5.828946,
- "implementation.": -5.828946,
- "importing": -5.828946,
- "in": -5.828946,
- "include": -4.730333,
- "indicates": -5.828946,
- "inheriting": -5.828946,
- "into": -4.037186,
- "is_first_line": -5.828946,
- "line": -5.828946,
- "lines": -4.442651,
- "loop": -5.828946,
- "message": -5.135798,
- "method": -5.135798,
- "methods": -5.135798,
- "msg.": -5.135798,
- "not": -4.730333,
- "of": -5.135798,
- "other": -4.730333,
- "parse": -5.135798,
- "pools": -5.828946,
- "pos": -5.135798,
- "private": -5.828946,
- "protected": -5.828946,
- "public": -4.730333,
- "raise": -5.828946,
- "raising": -5.828946,
- "ref": -5.828946,
- "returning.": -5.828946,
- "section.": -4.730333,
- "separator": -5.828946,
- "separator.": -5.828946,
- "skip_first_line": -5.828946,
- "skip_first_line.": -5.828946,
- "source": -4.730333,
- "split": -5.828946,
- "standard": -5.135798,
- "string": -5.828946,
- "string.": -4.730333,
- "super": -5.828946,
- "symbols": -5.828946,
- "table": -4.730333,
- "text_ended": -5.828946,
- "the": -4.730333,
- "to": -4.730333,
- "type": -3.431050,
- "value": -5.135798,
- "values": -5.135798,
- "|": -3.883035,
+ "!": -4.637712,
+ "\"": -7.202661,
+ "(": -5.816367,
+ ")": -5.816367,
+ "*": -3.619142,
+ "+": -5.005437,
+ "-": -0.322277,
+ ".": -5.005437,
+ "<-()]>": -6.509514,
+ "": -6.509514,
+ "": -5.816367,
+ "": -6.104049,
+ "=": -4.900076,
+ ">": -5.123220,
+ "ABAP_BOOL": -7.202661,
+ "C": -7.202661,
+ "CLASS": -6.509514,
+ "CL_CSV_PARSER": -5.410902,
+ "CONSTRUCTOR": -7.202661,
+ "CSV": -7.202661,
+ "CSVSTRING": -7.202661,
+ "CX": -7.202661,
+ "CX_CSV_PARSE_ERROR": -7.202661,
+ "DEFINITION": -6.509514,
+ "DELEGATE": -7.202661,
+ "Get": -7.202661,
+ "IF_CSV_PARSER_DELEGATE": -7.202661,
+ "IMPLEMENTATION": -6.509514,
+ "Instance": -6.509514,
+ "Method": -6.509514,
+ "Parse": -7.202661,
+ "Private": -7.202661,
+ "Public": -7.202661,
+ "REF": -7.202661,
+ "RETURNING": -7.202661,
+ "SEPARATOR": -7.202661,
+ "SIGNATURE": -6.509514,
+ "SKIP_FIRST_LINE": -7.202661,
+ "STRING": -7.202661,
+ "STRINGTAB": -6.104049,
+ "Space": -6.509514,
+ "TO": -7.202661,
+ "TYPE": -5.256751,
+ "This": -7.202661,
+ "[": -5.593223,
+ "]": -5.816367,
+ "_LINES": -7.202661,
+ "_csvstring": -6.509514,
+ "_delegate": -7.202661,
+ "_lines": -7.202661,
+ "_lines.": -7.202661,
+ "_parse_line": -6.509514,
+ "_separator": -7.202661,
+ "_skip_first_line": -7.202661,
+ "_textindicator": -7.202661,
+ "`": -5.816367,
+ "abap": -7.202661,
+ "abap_bool": -6.509514,
+ "abap_true.": -6.509514,
+ "an": -7.202661,
+ "append": -6.509514,
+ "assigning": -7.202661,
+ "at": -6.509514,
+ "c": -6.509514,
+ "char": -6.509514,
+ "cl_abap_char_utilities": -7.202661,
+ "cl_csv_parser": -6.509514,
+ "cl_object": -7.202661,
+ "class": -6.509514,
+ "clear": -7.202661,
+ "concatenate": -5.816367,
+ "constants": -7.202661,
+ "constructor": -6.509514,
+ "constructor.": -7.202661,
+ "cr_lf": -7.202661,
+ "create": -7.202661,
+ "csv": -7.202661,
+ "csvstring": -7.202661,
+ "csvstring.": -7.202661,
+ "csvvalue": -5.410902,
+ "csvvalue.": -5.593223,
+ "csvvalues.": -6.509514,
+ "cx_csv_parse_error": -6.509514,
+ "data": -6.104049,
+ "definition": -7.202661,
+ "delegate": -7.202661,
+ "delegate.": -7.202661,
+ "do": -6.104049,
+ "e": -7.202661,
+ "else.": -5.816367,
+ "endclass.": -7.202661,
+ "endif.": -5.410902,
+ "endmethod.": -6.509514,
+ "endwhile.": -6.509514,
+ "error": -7.202661,
+ "exception": -7.202661,
+ "exporting": -7.202661,
+ "field": -7.202661,
+ "files": -6.104049,
+ "final": -7.202661,
+ "formatting": -7.202661,
+ "from": -7.202661,
+ "here": -6.104049,
+ "if_csv_parser_delegate": -7.202661,
+ "implementation.": -7.202661,
+ "importing": -7.202661,
+ "in": -7.202661,
+ "include": -6.104049,
+ "indicates": -7.202661,
+ "inheriting": -7.202661,
+ "into": -5.410902,
+ "is_first_line": -7.202661,
+ "line": -7.202661,
+ "lines": -5.816367,
+ "loop": -7.202661,
+ "message": -6.509514,
+ "method": -6.509514,
+ "methods": -6.509514,
+ "msg.": -6.509514,
+ "not": -6.104049,
+ "of": -6.509514,
+ "other": -6.104049,
+ "parse": -6.509514,
+ "pools": -7.202661,
+ "pos": -6.509514,
+ "private": -7.202661,
+ "protected": -7.202661,
+ "public": -6.104049,
+ "raise": -7.202661,
+ "raising": -7.202661,
+ "ref": -7.202661,
+ "returning.": -7.202661,
+ "section.": -6.104049,
+ "separator": -7.202661,
+ "separator.": -7.202661,
+ "skip_first_line": -7.202661,
+ "skip_first_line.": -7.202661,
+ "source": -6.104049,
+ "split": -7.202661,
+ "standard": -6.509514,
+ "string": -7.202661,
+ "string.": -6.104049,
+ "super": -7.202661,
+ "symbols": -7.202661,
+ "table": -6.104049,
+ "text_ended": -7.202661,
+ "the": -6.104049,
+ "to": -6.104049,
+ "type": -4.804766,
+ "value": -6.509514,
+ "values": -6.509514,
+ "|": -5.256751,
},
"ABNF": map[string]float64{
- "(": -3.938340,
- ")": -4.056123,
- "*": -4.631487,
- "*DIGIT": -6.828712,
- "*basic": -6.135565,
- "*literal": -6.828712,
- "*newline": -6.828712,
- "*non": -6.828712,
- ",": -5.036953,
- "-": -1.576439,
- "/": -3.165150,
- ":": -5.442418,
- ";": -2.821379,
- "=": -2.434263,
- "ABNF": -6.135565,
- "ALPHA": -6.135565,
- "Array": -6.135565,
- "Basic": -5.730100,
- "Boolean": -6.828712,
- "Built": -6.828712,
- "Comment": -6.828712,
- "DIGIT": -4.120662,
- "Datetime": -6.828712,
- "Float": -6.828712,
- "HEXDIG": -6.828712,
- "Inline": -6.828712,
- "Integer": -6.828712,
- "Key": -6.828712,
- "License": -6.828712,
- "Literal": -6.135565,
- "MIT": -6.828712,
- "Multiline": -6.135565,
- "Newline": -6.828712,
- "RFC": -6.135565,
- "See": -6.828712,
- "Source": -6.828712,
- "Standard": -6.828712,
- "String": -5.219274,
- "Strings": -6.828712,
- "TOML": -6.135565,
- "Table": -5.442418,
- "This": -6.828712,
- "Value": -6.828712,
- "Whitespace": -6.828712,
- "[": -4.631487,
- "]": -4.631487,
- "according": -6.828712,
- "an": -6.828712,
- "apostraphe": -5.036953,
- "array": -3.938340,
- "as": -6.828712,
- "attempt": -6.828712,
- "based": -6.135565,
- "basic": -4.189655,
- "body": -5.442418,
- "boolean": -6.135565,
- "char": -4.749271,
- "clarity": -6.828712,
- "close": -4.749271,
- "comment": -4.749271,
- "date": -4.526127,
- "decimal": -6.135565,
- "define": -6.828712,
- "defined": -6.135565,
- "delim": -5.036953,
- "digit": -6.135565,
- "e": -6.135565,
- "empty": -5.730100,
- "eol": -6.135565,
- "escape": -6.135565,
- "escaped": -6.828712,
- "exp": -5.730100,
- "expression": -5.730100,
- "false": -6.135565,
- "float": -6.135565,
- "for": -6.828712,
- "frac": -5.730100,
- "full": -5.442418,
- "fullyear": -6.135565,
- "grammar": -6.828712,
- "here": -6.828712,
- "hour": -5.730100,
- "http": -6.828712,
- "https": -6.828712,
- "in": -5.442418,
- "inline": -4.263763,
- "int": -5.442418,
- "integer": -5.442418,
- "is": -6.828712,
- "key": -4.120662,
- "keyval": -5.036953,
- "keyvals": -5.219274,
- "leap": -6.828712,
- "literal": -4.343805,
- "mark": -4.749271,
- "mday": -6.135565,
- "minus": -6.135565,
- "minute": -5.730100,
- "ml": -3.832980,
- "month": -6.135565,
- "month/year": -6.828712,
- "newline": -5.219274,
- "newlines": -5.219274,
- "non": -5.442418,
- "numoffset": -6.135565,
- "offset": -6.135565,
- "on": -6.135565,
- "open": -4.749271,
- "pairs": -6.828712,
- "partial": -6.135565,
- "plus": -6.135565,
- "point": -6.135565,
- "prefixable": -6.135565,
- "quotation": -4.749271,
- "quoted": -6.135565,
- "reproduced": -6.828712,
- "rules": -6.828712,
- "secfrac": -6.135565,
- "second": -5.730100,
- "sep": -4.343805,
- "start": -6.135565,
- "std": -5.036953,
- "string": -4.056123,
- "symbol": -6.135565,
- "table": -3.427515,
- "terms": -6.828712,
- "the": -6.828712,
- "time": -3.832980,
- "to": -6.135565,
- "toml": -6.828712,
- "true": -6.135565,
- "underscore": -5.730100,
- "unescaped": -5.730100,
- "unquoted": -6.135565,
- "val": -5.036953,
- "values": -5.730100,
- "ws": -4.120662,
- "zero": -6.135565,
+ "%": -3.171745,
+ "'": -7.142037,
+ "(": -4.251665,
+ ")": -4.197598,
+ "*": -4.944812,
+ "*DIGIT": -7.142037,
+ "*basic": -6.448889,
+ "*literal": -7.142037,
+ "*newline": -7.142037,
+ "*non": -7.142037,
+ "+": -4.744141,
+ ",": -5.062595,
+ "-": -1.785450,
+ ".": -6.043424,
+ "/": -2.983153,
+ "//github.com/toml": -7.142037,
+ "//www.ietf.org/rfc/rfc": -7.142037,
+ ":": -5.755742,
+ ";": -2.588160,
+ "=": -2.747587,
+ "A": -5.196126,
+ "ABNF": -6.448889,
+ "ALPHA": -6.448889,
+ "Apostrophe": -7.142037,
+ "Array": -6.448889,
+ "B": -5.196126,
+ "B.": -7.142037,
+ "Basic": -6.043424,
+ "Boolean": -7.142037,
+ "Built": -7.142037,
+ "C": -5.350277,
+ "C.": -7.142037,
+ "CRLF": -7.142037,
+ "Comma": -6.448889,
+ "Comment": -7.142037,
+ "D": -4.839451,
+ "D.": -6.448889,
+ "DIGIT": -4.433986,
+ "Datetime": -7.142037,
+ "Double": -6.448889,
+ "E": -5.755742,
+ "Float": -7.142037,
+ "HEXDIG": -6.043424,
+ "Horizontal": -7.142037,
+ "Inline": -7.142037,
+ "Integer": -7.142037,
+ "Key": -7.142037,
+ "LF": -7.142037,
+ "Left": -7.142037,
+ "License": -7.142037,
+ "Literal": -6.448889,
+ "MIT": -7.142037,
+ "Multiline": -6.448889,
+ "Newline": -7.142037,
+ "Period": -7.142037,
+ "RFC": -6.448889,
+ "Right": -7.142037,
+ "See": -7.142037,
+ "Source": -7.142037,
+ "Space": -7.142037,
+ "Standard": -7.142037,
+ "String": -5.532599,
+ "Strings": -7.142037,
+ "TOML": -6.448889,
+ "Table": -5.755742,
+ "This": -7.142037,
+ "U": -4.839451,
+ "UXXXXXXXX": -7.142037,
+ "Value": -7.142037,
+ "Whitespace": -7.142037,
+ "XXXX": -7.142037,
+ "XXXXXXXX": -7.142037,
+ "Z": -6.448889,
+ "[": -4.577087,
+ "\\": -6.448889,
+ "]": -4.577087,
+ "_": -6.448889,
+ "a": -6.448889,
+ "according": -7.142037,
+ "an": -7.142037,
+ "apostraphe": -5.350277,
+ "array": -4.251665,
+ "as": -7.142037,
+ "attempt": -7.142037,
+ "b": -7.142037,
+ "backspace": -7.142037,
+ "based": -6.448889,
+ "basic": -4.502979,
+ "body": -5.755742,
+ "boolean": -6.448889,
+ "bracket": -5.755742,
+ "carriage": -7.142037,
+ "char": -5.062595,
+ "clarity": -7.142037,
+ "close": -5.062595,
+ "comment": -5.062595,
+ "date": -4.839451,
+ "decimal": -6.448889,
+ "define": -7.142037,
+ "defined": -6.448889,
+ "delim": -5.350277,
+ "digit": -6.448889,
+ "e": -6.043424,
+ "empty": -6.043424,
+ "eol": -7.142037,
+ "escape": -6.448889,
+ "escaped": -7.142037,
+ "exp": -6.043424,
+ "expression": -6.043424,
+ "f": -7.142037,
+ "false": -6.043424,
+ "feed": -6.448889,
+ "float": -6.448889,
+ "for": -7.142037,
+ "form": -7.142037,
+ "frac": -6.043424,
+ "full": -5.755742,
+ "fullyear": -6.448889,
+ "grammar": -7.142037,
+ "here": -7.142037,
+ "hour": -6.043424,
+ "http": -7.142037,
+ "https": -7.142037,
+ "in": -5.755742,
+ "inline": -4.577087,
+ "int": -5.755742,
+ "integer": -5.755742,
+ "is": -7.142037,
+ "key": -4.433986,
+ "keyval": -5.350277,
+ "keyvals": -5.532599,
+ "lang/toml": -7.142037,
+ "leap": -7.142037,
+ "left": -7.142037,
+ "line": -7.142037,
+ "literal": -4.657130,
+ "mark": -4.944812,
+ "mday": -6.448889,
+ "minus": -6.448889,
+ "minute": -6.043424,
+ "ml": -4.146304,
+ "month": -6.448889,
+ "month/year": -7.142037,
+ "n": -7.142037,
+ "newline": -5.532599,
+ "newlines": -5.532599,
+ "non": -6.043424,
+ "numoffset": -6.448889,
+ "offset": -6.448889,
+ "on": -6.448889,
+ "open": -5.062595,
+ "pairs": -7.142037,
+ "partial": -6.448889,
+ "plus": -6.448889,
+ "point": -6.448889,
+ "prefixable": -6.448889,
+ "quotation": -4.944812,
+ "quoted": -6.448889,
+ "r": -7.142037,
+ "reproduced": -7.142037,
+ "return": -7.142037,
+ "reverse": -7.142037,
+ "right": -7.142037,
+ "rules": -7.142037,
+ "secfrac": -6.448889,
+ "second": -6.043424,
+ "sep": -4.657130,
+ "solidus": -6.448889,
+ "square": -5.755742,
+ "start": -6.448889,
+ "std": -5.350277,
+ "string": -4.369448,
+ "symbol": -6.448889,
+ "t": -7.142037,
+ "tab": -6.448889,
+ "table": -3.740839,
+ "terms": -7.142037,
+ "the": -7.142037,
+ "time": -4.146304,
+ "to": -6.448889,
+ "toml": -7.142037,
+ "true": -6.043424,
+ "txt": -7.142037,
+ "uXXXX": -7.142037,
+ "underscore": -6.043424,
+ "unescaped": -6.043424,
+ "unquoted": -6.448889,
+ "val": -5.350277,
+ "values": -6.043424,
+ "ws": -4.006542,
+ "x": -3.171745,
+ "z": -6.448889,
+ "zero": -6.448889,
+ "{": -7.142037,
+ "}": -7.142037,
},
"AGS Script": map[string]float64{
- "!": -5.422745,
- "&&": -5.710427,
- "(": -2.184066,
- ")": -2.184066,
- "*control": -4.283311,
- "*theGui": -7.502186,
- "+": -5.556276,
- ",": -3.610366,
- "-": -5.017280,
- ":": -6.809039,
- ";": -2.354692,
- "<": -7.502186,
- "=": -2.169468,
- ">": -6.809039,
- "DISTANCE": -4.324133,
- "Debug": -6.115892,
- "DeleteSaveSlot": -7.502186,
- "Display": -6.115892,
- "EventType": -7.502186,
- "GUI": -7.502186,
- "GUIControl": -4.283311,
- "IsGamePaused": -6.403574,
- "IsInterfaceEnabled": -6.809039,
- "IsKeyPressed": -4.668973,
- "IsSpeechVoxAvailable": -7.502186,
- "KeyboardMovement": -6.809039,
- "KeyboardMovement_CurrentDirection": -5.556276,
- "KeyboardMovement_Directions": -6.115892,
- "KeyboardMovement_KeyDown": -5.892749,
- "KeyboardMovement_KeyDownLeft": -6.403574,
- "KeyboardMovement_KeyDownRight": -6.403574,
- "KeyboardMovement_KeyLeft": -5.892749,
- "KeyboardMovement_KeyRight": -5.892749,
- "KeyboardMovement_KeyStop": -6.403574,
- "KeyboardMovement_KeyUp": -5.892749,
- "KeyboardMovement_KeyUpLeft": -6.403574,
- "KeyboardMovement_KeyUpRight": -6.403574,
- "KeyboardMovement_Mode": -6.115892,
- "KeyboardMovement_Modes": -6.115892,
- "MouseButton": -4.411144,
- "ProcessClick": -7.502186,
- "QuitGame": -6.809039,
- "RestoreGameSlot": -7.502186,
- "SaveGameSlot": -7.502186,
- "SetMode": -6.809039,
- "SetVoiceMode": -6.115892,
- "System.SupportsGammaControl": -7.502186,
- "System.Volume": -7.502186,
- "Wait": -6.403574,
- "[": -5.710427,
- "]": -5.710427,
- "btnAbout_OnClick": -7.502186,
- "btnCancelRestore_OnClick": -7.502186,
- "btnCancelSave_OnClick": -7.502186,
- "btnDeleteSave_OnClick": -7.502186,
- "btnIconAbout_Click": -7.502186,
- "btnIconCurInv_Click": -7.502186,
- "btnIconExit_Click": -7.502186,
- "btnIconInv_Click": -7.502186,
- "btnIconLoad": -7.502186,
- "btnIconLoad_Click": -6.809039,
- "btnIconSave": -7.502186,
- "btnIconSave_Click": -6.809039,
- "btnInvDown_Click": -7.502186,
- "btnInvOK_Click": -7.502186,
- "btnInvSelect_Click": -7.502186,
- "btnInvUp_Click": -7.502186,
- "btnLoad_OnClick": -7.502186,
- "btnQuit_OnClick": -7.502186,
- "btnRestoreGame_OnClick": -7.502186,
- "btnResume_OnClick": -7.502186,
- "btnSaveGame_OnClick": -6.809039,
- "btnSave_OnClick": -7.502186,
- "btnVoice.Text": -5.556276,
- "btnVoice.Visible": -7.502186,
- "button": -4.324133,
- "cEgo_Interact": -7.502186,
- "cEgo_Look": -7.502186,
- "cEgo_Talk": -7.502186,
- "close_restore_game_dialog": -6.403574,
- "close_save_game_dialog": -6.403574,
- "control": -7.502186,
- "data": -7.502186,
- "dx": -4.506454,
- "dy": -4.506454,
- "eEventLeaveRoom": -7.502186,
- "eKeyCtrlA": -7.502186,
- "eKeyCtrlS": -7.502186,
- "eKeyCtrlV": -7.502186,
- "eKeyCtrlW": -7.502186,
- "eKeyCtrlX": -7.502186,
- "eKeyEscape": -7.502186,
- "eKeyboardMovement_Down": -5.892749,
- "eKeyboardMovement_DownLeft": -5.892749,
- "eKeyboardMovement_DownRight": -5.892749,
- "eKeyboardMovement_Left": -5.892749,
- "eKeyboardMovement_None": -6.809039,
- "eKeyboardMovement_Pressing": -6.809039,
- "eKeyboardMovement_Right": -5.892749,
- "eKeyboardMovement_Stop": -5.304962,
- "eKeyboardMovement_Tapping": -6.809039,
- "eKeyboardMovement_Up": -5.892749,
- "eKeyboardMovement_UpLeft": -5.892749,
- "eKeyboardMovement_UpRight": -5.892749,
- "eModeInteract": -7.502186,
- "eModePointer": -5.892749,
- "eModeTalkto": -7.502186,
- "eModeUseinv": -6.809039,
- "eModeWalkto": -7.502186,
- "eMouseLeft": -6.403574,
- "eMouseWheelNorth": -7.502186,
- "eNoBlock": -6.809039,
- "eSpeechTextOnly": -7.502186,
- "eSpeechVoiceAndText": -6.809039,
- "eSpeechVoiceOnly": -7.502186,
- "else": -3.864600,
- "enum": -6.809039,
- "event": -6.809039,
- "false": -4.557748,
- "function": -3.813307,
- "gControl_OnClick": -7.502186,
- "gIconbar.Visible": -5.199601,
- "gInventory.Visible": -7.502186,
- "gPanel.Centre": -7.502186,
- "gPanel.Visible": -5.556276,
- "gRestartYN.Centre": -7.502186,
- "gRestartYN.Visible": -6.809039,
- "gRestoreGame.Visible": -6.809039,
- "gSaveGame.Visible": -7.502186,
- "game.debug_mode": -7.502186,
- "gameSlotToSaveInto": -6.403574,
- "i": -5.892749,
- "if": -3.407842,
- "import": -7.502186,
- "initialize_control_panel": -7.502186,
- "int": -4.668973,
- "interface": -7.502186,
- "interface_click": -7.502186,
- "invCustomInv.ScrollDown": -7.502186,
- "invCustomInv.ScrollUp": -7.502186,
- "keycode": -4.668973,
- "lblVoice.Visible": -7.502186,
- "lstRestoreGamesList.FillSaveGameList": -7.502186,
- "lstRestoreGamesList.SaveGameSlots": -7.502186,
- "lstRestoreGamesList.SelectedIndex": -6.809039,
- "lstSaveGamesList.FillSaveGameList": -7.502186,
- "lstSaveGamesList.ItemCount": -6.809039,
- "lstSaveGamesList.Items": -6.403574,
- "lstSaveGamesList.SaveGameSlots": -6.809039,
- "lstSaveGamesList.SelectedIndex": -6.403574,
- "lstSaveGamesList_OnSelectionCh": -7.502186,
- "mode": -6.403574,
- "mouse.Mode": -6.115892,
- "mouse.UseDefaultGraphic": -5.556276,
- "mouse.UseModeGraphic": -5.892749,
- "mouse.x": -7.502186,
- "mouse.y": -7.502186,
- "newdirection": -3.740986,
- "null": -7.502186,
- "on_event": -7.502186,
- "on_key_press": -7.502186,
- "on_mouse_click": -7.502186,
- "player.ActiveInventory": -7.502186,
- "player.PlaceOnWalkableArea": -7.502186,
- "player.StopMoving": -6.403574,
- "player.WalkStraight": -6.809039,
- "player.on": -6.809039,
- "player.x": -6.809039,
- "player.y": -6.809039,
- "repeatedly_execute": -7.502186,
- "return": -6.809039,
- "show_inventory_window": -7.502186,
- "show_restore_game_dialog": -6.809039,
- "show_save_game_dialog": -7.502186,
- "sldAudio.Value": -7.502186,
- "sldAudio_OnChange": -7.502186,
- "sldGamma_OnChange": -7.502186,
- "sldVoice.Visible": -7.502186,
- "sldVoice_OnChange": -7.502186,
- "static": -6.809039,
- "struct": -7.502186,
- "true": -5.104291,
- "txtNewSaveName.Text": -5.892749,
- "txtNewSaveName_OnActivate": -7.502186,
- "while": -7.502186,
- "{": -3.158381,
- "||": -5.199601,
- "}": -3.132739,
+ "!": -5.850405,
+ "#define": -7.236699,
+ "&&": -6.138087,
+ "(": -2.606836,
+ ")": -2.606836,
+ "*control": -4.710971,
+ "*theGui": -7.929846,
+ "+": -5.983936,
+ ",": -3.978603,
+ "-": -2.601970,
+ "//": -5.531951,
+ "//****************************************************************************************************": -5.850405,
+ "//But": -7.929846,
+ "//Ctrl": -7.929846,
+ "//START": -7.929846,
+ "//Use": -7.929846,
+ "//if": -7.929846,
+ ":": -7.236699,
+ ";": -2.776555,
+ "<": -7.929846,
+ "=": -1.114206,
+ ">": -7.236699,
+ "CONTROL": -7.929846,
+ "DISTANCE": -4.710971,
+ "Debug": -6.543552,
+ "DeleteSaveSlot": -7.929846,
+ "Display": -6.543552,
+ "ESC": -7.929846,
+ "EventType": -7.929846,
+ "FUNCTIONS": -7.929846,
+ "GUI": -7.929846,
+ "GUIControl": -4.710971,
+ "IsGamePaused": -6.831234,
+ "IsInterfaceEnabled": -7.236699,
+ "IsKeyPressed": -5.096633,
+ "IsSpeechVoxAvailable": -7.929846,
+ "KeyboardMovement": -7.236699,
+ "KeyboardMovement_CurrentDirection": -5.983936,
+ "KeyboardMovement_Directions": -6.543552,
+ "KeyboardMovement_KeyDown": -6.320409,
+ "KeyboardMovement_KeyDownLeft": -6.831234,
+ "KeyboardMovement_KeyDownRight": -6.831234,
+ "KeyboardMovement_KeyLeft": -6.320409,
+ "KeyboardMovement_KeyRight": -6.320409,
+ "KeyboardMovement_KeyStop": -6.831234,
+ "KeyboardMovement_KeyUp": -6.320409,
+ "KeyboardMovement_KeyUpLeft": -6.831234,
+ "KeyboardMovement_KeyUpRight": -6.831234,
+ "KeyboardMovement_Mode": -6.543552,
+ "KeyboardMovement_Modes": -6.543552,
+ "KeyboardMovement_VERSION": -7.929846,
+ "MouseButton": -4.838804,
+ "OF": -7.929846,
+ "PANEL": -7.929846,
+ "ProcessClick": -7.929846,
+ "QuitGame": -7.236699,
+ "RestoreGameSlot": -7.929846,
+ "SaveGameSlot": -7.929846,
+ "SetMode": -7.236699,
+ "SetVoiceMode": -6.543552,
+ "System.SupportsGammaControl": -7.929846,
+ "System.Volume": -7.929846,
+ "W": -7.929846,
+ "Wait": -6.831234,
+ "[": -6.138087,
+ "]": -6.138087,
+ "appreciate.": -7.929846,
+ "area": -7.929846,
+ "as": -7.929846,
+ "btnAbout_OnClick": -7.929846,
+ "btnCancelRestore_OnClick": -7.929846,
+ "btnCancelSave_OnClick": -7.929846,
+ "btnDeleteSave_OnClick": -7.929846,
+ "btnIconAbout_Click": -7.929846,
+ "btnIconCurInv_Click": -7.929846,
+ "btnIconExit_Click": -7.929846,
+ "btnIconInv_Click": -7.929846,
+ "btnIconLoad": -7.929846,
+ "btnIconLoad_Click": -7.236699,
+ "btnIconSave": -7.929846,
+ "btnIconSave_Click": -7.236699,
+ "btnInvDown_Click": -7.929846,
+ "btnInvOK_Click": -7.929846,
+ "btnInvSelect_Click": -7.929846,
+ "btnInvUp_Click": -7.929846,
+ "btnLoad_OnClick": -7.929846,
+ "btnQuit_OnClick": -7.929846,
+ "btnRestoreGame_OnClick": -7.929846,
+ "btnResume_OnClick": -7.929846,
+ "btnSaveGame_OnClick": -7.236699,
+ "btnSave_OnClick": -7.929846,
+ "btnVoice.Text": -5.983936,
+ "btnVoice.Visible": -7.929846,
+ "button": -4.751793,
+ "cEgo_Interact": -7.929846,
+ "cEgo_Look": -7.929846,
+ "cEgo_Talk": -7.929846,
+ "cancel": -7.929846,
+ "close_restore_game_dialog": -6.831234,
+ "close_save_game_dialog": -6.831234,
+ "control": -7.929846,
+ "data": -7.929846,
+ "dx": -4.934114,
+ "dy": -4.934114,
+ "eEventLeaveRoom": -7.929846,
+ "eKeyCtrlA": -7.929846,
+ "eKeyCtrlS": -7.929846,
+ "eKeyCtrlV": -7.929846,
+ "eKeyCtrlW": -7.929846,
+ "eKeyCtrlX": -7.929846,
+ "eKeyEscape": -7.929846,
+ "eKeyboardMovement_Down": -6.320409,
+ "eKeyboardMovement_DownLeft": -6.320409,
+ "eKeyboardMovement_DownRight": -6.320409,
+ "eKeyboardMovement_Left": -6.320409,
+ "eKeyboardMovement_None": -7.236699,
+ "eKeyboardMovement_Pressing": -7.236699,
+ "eKeyboardMovement_Right": -6.320409,
+ "eKeyboardMovement_Stop": -5.732622,
+ "eKeyboardMovement_Tapping": -7.236699,
+ "eKeyboardMovement_Up": -6.320409,
+ "eKeyboardMovement_UpLeft": -6.320409,
+ "eKeyboardMovement_UpRight": -6.320409,
+ "eModeInteract": -7.929846,
+ "eModePickup": -7.929846,
+ "eModePointer": -6.320409,
+ "eModeTalkto": -7.929846,
+ "eModeUseinv": -7.236699,
+ "eModeWalkto": -7.929846,
+ "eMouseLeft": -6.831234,
+ "eMouseWheelNorth": -7.929846,
+ "eNoBlock": -7.236699,
+ "eSpeechTextOnly": -7.929846,
+ "eSpeechVoiceAndText": -7.236699,
+ "eSpeechVoiceOnly": -7.929846,
+ "else": -4.292260,
+ "enum": -7.236699,
+ "event": -7.236699,
+ "false": -4.985407,
+ "function": -4.240967,
+ "gControl_OnClick": -7.929846,
+ "gIconbar.Visible": -5.627261,
+ "gInventory.Visible": -7.929846,
+ "gPanel.Centre": -7.929846,
+ "gPanel.Visible": -5.983936,
+ "gRestartYN.Centre": -7.929846,
+ "gRestartYN.Visible": -7.236699,
+ "gRestoreGame.Visible": -7.236699,
+ "gSaveGame.Visible": -7.929846,
+ "game.debug_mode": -7.929846,
+ "gameSlotToSaveInto": -6.831234,
+ "give": -7.929846,
+ "i": -6.320409,
+ "if": -3.835502,
+ "import": -7.929846,
+ "initialize_control_panel": -7.929846,
+ "int": -5.096633,
+ "interface": -7.929846,
+ "interface_click": -7.929846,
+ "invCustomInv.ScrollDown": -7.929846,
+ "invCustomInv.ScrollUp": -7.929846,
+ "is": -7.929846,
+ "keyboard": -7.929846,
+ "keycode": -4.985407,
+ "lblVoice.Visible": -7.929846,
+ "lstRestoreGamesList.FillSaveGameList": -7.929846,
+ "lstRestoreGamesList.SaveGameSlots": -7.929846,
+ "lstRestoreGamesList.SelectedIndex": -7.236699,
+ "lstSaveGamesList.FillSaveGameList": -7.929846,
+ "lstSaveGamesList.ItemCount": -7.236699,
+ "lstSaveGamesList.Items": -6.831234,
+ "lstSaveGamesList.SaveGameSlots": -7.236699,
+ "lstSaveGamesList.SelectedIndex": -6.831234,
+ "lstSaveGamesList_OnSelectionCh": -7.929846,
+ "mode": -6.831234,
+ "mouse.Mode": -6.320409,
+ "mouse.UseDefaultGraphic": -5.983936,
+ "mouse.UseModeGraphic": -6.320409,
+ "mouse.x": -7.929846,
+ "mouse.y": -7.929846,
+ "move": -7.929846,
+ "much": -7.929846,
+ "newdirection": -4.168646,
+ "null": -7.929846,
+ "on_event": -7.929846,
+ "on_key_press": -7.929846,
+ "on_mouse_click": -7.929846,
+ "player.ActiveInventory": -7.929846,
+ "player.PlaceOnWalkableArea": -7.929846,
+ "player.StopMoving": -6.831234,
+ "player.WalkStraight": -7.236699,
+ "player.on": -7.236699,
+ "player.x": -7.236699,
+ "player.y": -7.236699,
+ "players": -7.929846,
+ "repeatedly_execute": -7.929846,
+ "restart.": -7.929846,
+ "return": -7.236699,
+ "shortcuts": -7.929846,
+ "show_inventory_window": -7.929846,
+ "show_restore_game_dialog": -7.236699,
+ "show_save_game_dialog": -7.929846,
+ "sldAudio.Value": -7.929846,
+ "sldAudio_OnChange": -7.929846,
+ "sldGamma_OnChange": -7.929846,
+ "sldVoice.Visible": -7.929846,
+ "sldVoice_OnChange": -7.929846,
+ "some": -7.929846,
+ "standard": -7.929846,
+ "static": -7.236699,
+ "struct": -7.929846,
+ "this": -7.929846,
+ "to": -7.236699,
+ "true": -5.531951,
+ "txtNewSaveName.Text": -6.320409,
+ "txtNewSaveName_OnActivate": -7.929846,
+ "very": -7.929846,
+ "walkable": -7.929846,
+ "while": -7.929846,
+ "will": -7.236699,
+ "you": -7.929846,
+ "your": -7.929846,
+ "{": -3.586041,
+ "||": -5.531951,
+ "}": -3.560399,
},
"AMPL": map[string]float64{
"(": -3.988984,
@@ -1002,416 +1111,460 @@ var DefaultClassifier Classifier = &classifier{
"}": -2.932931,
},
"API Blueprint": map[string]float64{
- "!": -4.960511,
- "'": -6.059123,
- "(": -2.840247,
- ")": -3.420066,
- "**": -6.059123,
- "**API": -6.059123,
- "**Note": -6.059123,
- "**parse**": -6.059123,
- "**real": -6.059123,
- "+": -2.840247,
- ",": -3.351073,
- "-": -4.113213,
- ":": -2.532763,
- "A": -4.267364,
- "API": -4.113213,
- "APIs": -6.059123,
- "Actions": -5.365976,
- "Advanced": -5.365976,
- "Also": -6.059123,
- "As": -6.059123,
- "Attributes": -4.960511,
- "Blueprint": -4.672829,
- "Blueprint**": -6.059123,
- "Blueprint**.": -6.059123,
- "Body": -6.059123,
- "Date": -6.059123,
- "FORMAT": -4.960511,
- "Hello": -6.059123,
- "ID.": -6.059123,
- "In": -6.059123,
- "JSON": -6.059123,
- "Model": -6.059123,
- "Next": -5.365976,
- "One": -6.059123,
- "Parameters": -4.672829,
- "Previous": -5.365976,
- "Raw": -4.672829,
- "Resource": -4.960511,
- "Response": -4.449685,
- "Retrieves": -6.059123,
- "The": -6.059123,
- "This": -4.113213,
- "Time": -6.059123,
- "We": -6.059123,
- "World": -6.059123,
- "[": -3.574217,
- "]": -3.574217,
- "a": -4.113213,
- "about": -6.059123,
- "action": -5.365976,
- "after": -6.059123,
- "also": -6.059123,
- "amount": -6.059123,
- "an": -6.059123,
- "and": -4.267364,
- "another": -5.365976,
- "application/json": -4.960511,
- "apply": -6.059123,
- "apply.": -6.059123,
- "as": -6.059123,
- "attributes": -6.059123,
- "be": -6.059123,
- "between": -6.059123,
- "body": -5.365976,
- "can": -5.365976,
- "case": -6.059123,
- "combined": -6.059123,
- "complementary": -6.059123,
- "contains": -6.059123,
- "coupon": -4.672829,
- "course": -6.059123,
- "created": -6.059123,
- "customer.": -6.059123,
- "demonstrates": -5.365976,
- "describe": -6.059123,
- "description": -6.059123,
- "discount": -5.365976,
- "do": -6.059123,
- "duplicate": -6.059123,
- "every": -6.059123,
- "example": -4.672829,
- "examples": -6.059123,
- "explain": -6.059123,
- "fact": -6.059123,
- "false": -5.365976,
- "forget": -6.059123,
- "given": -6.059123,
- "going": -6.059123,
- "how": -6.059123,
- "https": -4.449685,
- "id": -4.960511,
- "in": -4.267364,
- "information": -6.059123,
- "installment": -6.059123,
- "integer": -6.059123,
- "is": -4.267364,
- "it": -5.365976,
- "keep": -6.059123,
- "longer": -6.059123,
- "message.": -6.059123,
- "method": -6.059123,
- "might": -6.059123,
- "mind": -6.059123,
- "next": -6.059123,
- "no": -6.059123,
- "not": -6.059123,
- "null": -6.059123,
- "number": -4.672829,
- "object": -6.059123,
- "of": -5.365976,
- "off": -5.365976,
- "on": -6.059123,
- "one": -6.059123,
- "or": -5.365976,
- "parser": -6.059123,
- "percent": -6.059123,
- "percent_off": -6.059123,
- "plain": -6.059123,
- "please": -6.059123,
- "positive": -6.059123,
- "priority": -6.059123,
- "progress": -6.059123,
- "provided": -6.059123,
- "redeem_by": -6.059123,
- "redeemed": -6.059123,
- "represents": -6.059123,
- "request": -6.059123,
- "resource": -4.960511,
- "resource.": -6.059123,
- "response": -6.059123,
- "s": -6.059123,
- "section.": -6.059123,
- "simplest": -6.059123,
- "single": -6.059123,
- "stamp": -6.059123,
- "state": -4.960511,
- "status": -6.059123,
- "string": -4.672829,
- "such": -6.059123,
- "text/plain": -6.059123,
- "that": -4.960511,
- "the": -3.420066,
- "this": -5.365976,
- "through": -6.059123,
- "to": -4.113213,
- "transition": -5.365976,
- "transition.": -6.059123,
- "true": -6.059123,
- "view": -6.059123,
- "want": -6.059123,
- "we": -6.059123,
- "what": -6.059123,
- "which": -6.059123,
- "will": -5.365976,
- "with": -4.960511,
- "written": -6.059123,
- "you": -5.365976,
- "{": -4.672829,
- "}": -4.672829,
- "–": -5.365976,
+ "!": -5.216746,
+ "#": -4.369448,
+ "##": -6.315358,
+ "%": -3.319626,
+ "'": -6.315358,
+ "(": -3.057261,
+ ")": -3.057261,
+ "**": -6.315358,
+ "**API": -6.315358,
+ "**Note": -6.315358,
+ "**parse**": -6.315358,
+ "**real": -6.315358,
+ "+": -3.096482,
+ ",": -3.542769,
+ "-": -3.917463,
+ ".": -5.622211,
+ "//github.com/apiaryio/drafter": -6.315358,
+ "//github.com/apiaryio/drafter#bindings": -6.315358,
+ "//raw.github.com/apiaryio/api": -4.929064,
+ ":": -2.760010,
+ "A": -4.523599,
+ "API": -4.118133,
+ "API.md": -5.622211,
+ "APIs": -6.315358,
+ "Action.md": -6.315358,
+ "Actions": -5.622211,
+ "Actions.md": -5.622211,
+ "Advanced": -4.705920,
+ "Also": -6.315358,
+ "As": -6.315358,
+ "Attributes": -5.216746,
+ "Attributes.md": -5.216746,
+ "Blueprint": -4.705920,
+ "Blueprint**": -6.315358,
+ "Blueprint**.": -6.315358,
+ "Body": -6.315358,
+ "Date": -6.315358,
+ "FORMAT": -5.216746,
+ "Github": -6.315358,
+ "Hello": -6.315358,
+ "ID.": -6.315358,
+ "In": -6.315358,
+ "JSON": -6.315358,
+ "Markdown": -6.315358,
+ "Model": -6.315358,
+ "Model.md": -6.315358,
+ "Next": -5.622211,
+ "One": -6.315358,
+ "Parameters": -4.929064,
+ "Parameters.md": -6.315358,
+ "Previous": -5.622211,
+ "Raw": -4.929064,
+ "Resource": -4.523599,
+ "Response": -4.705920,
+ "Retrieves": -6.315358,
+ "Simplest": -5.622211,
+ "The": -6.315358,
+ "This": -4.369448,
+ "Time": -6.315358,
+ "We": -6.315358,
+ "World": -6.315358,
+ "[": -3.750409,
+ "]": -3.750409,
+ "a": -4.369448,
+ "about": -6.315358,
+ "action": -5.622211,
+ "after": -6.315358,
+ "also": -6.315358,
+ "amount": -6.315358,
+ "an": -6.315358,
+ "and": -4.118133,
+ "another": -5.622211,
+ "application/json": -5.216746,
+ "apply": -6.315358,
+ "apply.": -6.315358,
+ "as": -5.622211,
+ "attributes": -5.622211,
+ "avoid": -6.315358,
+ "be": -6.315358,
+ "between": -6.315358,
+ "bindings": -6.315358,
+ "blueprint/master/examples/": -4.929064,
+ "body": -5.622211,
+ "can": -5.622211,
+ "case": -6.315358,
+ "code": -6.315358,
+ "combined": -6.315358,
+ "complementary": -6.315358,
+ "contains": -6.315358,
+ "coupon": -4.929064,
+ "course": -6.315358,
+ "created": -6.315358,
+ "customer.": -6.315358,
+ "demonstrate": -6.315358,
+ "demonstrates": -5.622211,
+ "describe": -6.315358,
+ "description": -6.315358,
+ "descriptions.": -6.315358,
+ "discount": -5.622211,
+ "do": -6.315358,
+ "duplicate": -6.315358,
+ "duplicates": -6.315358,
+ "every": -6.315358,
+ "example": -4.705920,
+ "examples": -6.315358,
+ "explain": -6.315358,
+ "fact": -6.315358,
+ "false": -5.622211,
+ "forget": -6.315358,
+ "given": -6.315358,
+ "going": -5.622211,
+ "how": -5.216746,
+ "https": -4.523599,
+ "id": -5.216746,
+ "in": -4.369448,
+ "information": -6.315358,
+ "installment": -6.315358,
+ "integer": -6.315358,
+ "is": -4.369448,
+ "it": -5.622211,
+ "its": -6.315358,
+ "just": -6.315358,
+ "keep": -6.315358,
+ "longer": -6.315358,
+ "message.": -6.315358,
+ "method": -6.315358,
+ "might": -6.315358,
+ "mind": -6.315358,
+ "next": -6.315358,
+ "no": -6.315358,
+ "not": -6.315358,
+ "null": -6.315358,
+ "number": -4.929064,
+ "object": -6.315358,
+ "of": -4.929064,
+ "off": -5.622211,
+ "on": -5.622211,
+ "one": -5.622211,
+ "opposed": -6.315358,
+ "or": -5.216746,
+ "output": -6.315358,
+ "parser": -6.315358,
+ "parser.": -6.315358,
+ "percent": -6.315358,
+ "percent_off": -6.315358,
+ "plain": -6.315358,
+ "please": -6.315358,
+ "positive": -6.315358,
+ "priority": -6.315358,
+ "progress": -6.315358,
+ "provided": -6.315358,
+ "really": -6.315358,
+ "redeem_by": -6.315358,
+ "redeemed": -6.315358,
+ "represents": -6.315358,
+ "request": -6.315358,
+ "resource": -5.216746,
+ "resource.": -6.315358,
+ "response": -6.315358,
+ "reuse": -6.315358,
+ "s": -6.315358,
+ "section.": -6.315358,
+ "see": -6.315358,
+ "seeing": -6.315358,
+ "simplest": -6.315358,
+ "single": -6.315358,
+ "stamp": -6.315358,
+ "state": -5.216746,
+ "status": -6.315358,
+ "string": -4.929064,
+ "such": -6.315358,
+ "text/plain": -6.315358,
+ "that": -5.216746,
+ "the": -3.482145,
+ "this": -5.622211,
+ "through": -6.315358,
+ "to": -3.917463,
+ "transition": -5.622211,
+ "transition.": -6.315358,
+ "true": -6.315358,
+ "view": -6.315358,
+ "want": -6.315358,
+ "we": -6.315358,
+ "what": -5.622211,
+ "which": -6.315358,
+ "will": -5.216746,
+ "with": -5.216746,
+ "written": -6.315358,
+ "you": -5.622211,
+ "{": -4.929064,
+ "}": -4.929064,
+ "–": -5.622211,
},
"APL": map[string]float64{
- "(": -3.506736,
- ")": -3.489037,
- "+": -5.047181,
- ",": -3.101271,
- "-": -5.586178,
- "/": -4.893031,
- "/C": -7.532088,
- "/Functions": -7.532088,
- "/Page": -7.532088,
- "/Z": -7.532088,
- "/config": -7.532088,
- "/input": -7.532088,
- "/lines": -7.532088,
- ":": -3.327396,
- ";": -3.327396,
- "": -7.532088,
- "