Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,4 +37,5 @@ jobs:
node-version: ${{ matrix.node-version }}

- run: npm ci
- run: npm run typecheck
- run: npm run test:all
22 changes: 21 additions & 1 deletion lib/cache.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,21 @@ class Cache {
this.cacheFolder = path.join(config.cache, 'cache');
}

/**
* Fetch stats from the cache folder for getting its last modified time
* (mtime).
*
* @returns {Promise<any>} A promise with the stats of the cache folder.
*/
lastUpdated() {
return fs.stat(this.cacheFolder);
}

/**
* Fetch a page from cache using preferred language and preferred platform.
* @param {string} page
* @returns {Promise<string>}
*/
getPage(page) {
let preferredPlatform = platforms.getPreferredPlatformFolder(this.config);
const preferredLanguage = process.env.LANG || 'en';
Expand All @@ -34,10 +45,20 @@ class Cache {
});
}

/**
* Clean the cache folder.
* @returns {Promise<any>} A promise when the remove is completed.
*/
clear() {
return fs.remove(this.cacheFolder);
}

/**
* Update the cache folder using a temporary directory, update the index and
* return it.
*
* @returns {Promise<any>} The index.
*/
update() {
// Temporary folder path: /tmp/tldr/{randomName}
const tempFolder = path.join(os.tmpdir(), 'tldr', utils.uniqueId());
Expand Down Expand Up @@ -65,7 +86,6 @@ class Cache {
index.rebuildPagesIndex(),
]);
})

.then(([_, shortIndex]) => {
return shortIndex;
});
Expand Down
4 changes: 2 additions & 2 deletions lib/completion.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class Completion {

appendScript(script) {
const rcFilePath = this.getFilePath();
return new Promise((resolve, reject) => {
return new Promise((/** @type {(v?: never) => void} */ resolve, reject) => {
fs.appendFile(rcFilePath, `\n${script}\n`, (err) => {
if (err) {
reject((new CompletionScriptError(`Error appending to ${rcFilePath}: ${err.message}`)));
Expand Down Expand Up @@ -79,4 +79,4 @@ fi
}
}

module.exports = Completion;
module.exports = Completion;
4 changes: 2 additions & 2 deletions lib/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ exports.get = () => {
const DEFAULT = path.join(__dirname, '..', 'config.json');
const CUSTOM = path.join(osHomedir(), '.tldrrc');

let defaultConfig = JSON.parse(fs.readFileSync(DEFAULT));
let defaultConfig = JSON.parse(fs.readFileSync(DEFAULT, 'utf-8'));
defaultConfig.cache = path.join(osHomedir(), '.tldr');

let customConfig = {};
try {
customConfig = JSON.parse(fs.readFileSync(CUSTOM));
customConfig = JSON.parse(fs.readFileSync(CUSTOM, 'utf-8'));
} catch (ex) {
if (ex instanceof SyntaxError) {
throw new Error('The content of .tldrrc is not a valid JSON object:\n' + ex);
Expand Down
61 changes: 47 additions & 14 deletions lib/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ let shortIndex = null;
const pagesPath = path.join(config.get().cache, 'cache');
const shortIndexFile = path.join(pagesPath, 'shortIndex.json');

/**
* @param {string} page
* @param {string|undefined} preferredPlatform
* @param {string} preferredLanguage
* @returns {Promise<?string>}
*/
function findPage(page, preferredPlatform, preferredLanguage) {
// Load the index
return getShortIndex()
Expand All @@ -33,7 +39,7 @@ function findPage(page, preferredPlatform, preferredLanguage) {
ll = preferredLanguage.substring(0, preferredLanguage.indexOf('_'));
}
if (!hasLang(targets, preferredLanguage)) {
preferredLanguage = ll;
preferredLanguage = /** @type {string} */ (ll);
}

// Page resolution logic:
Expand Down Expand Up @@ -91,25 +97,38 @@ function hasLang(targets, preferredLanguage) {
});
}

// hasPage is always called after the index is created,
// hence just return the variable in memory.
// There is no need to re-read the index file again.
/**
* Check if a page is in the index.
*
* @returns {boolean} The presence of the page in the index.
*/
function hasPage(page) {
// hasPage is always called after the index is created,
// hence just return the variable in memory.
// There is no need to re-read the index file again.
if (!shortIndex) {
return false;
}
return page in shortIndex;
}

// Return all commands available in the local cache.
/**
* Return all commands available in the local index.
* @returns {Promise<string[]>} A promise with the commands from the index.
*/
function commands() {
return getShortIndex().then((idx) => {
return Object.keys(idx).sort();
});
}

// Return all commands for a given platform.
// P.S. - The platform 'common' is always included.
/**
* Return all commands for a given platform. The 'common' platform is always
* included.
*
* @param {string} platform The desired platform.
* @returns {Promise<string[]>} The commands for a given platform.
*/
function commandsFor(platform) {
return getShortIndex()
.then((idx) => {
Expand All @@ -124,7 +143,11 @@ function commandsFor(platform) {
});
}

// Delete the index file.
/**
* Delete the index file.
*
* @returns {Promise<any>} A promise when the remove is completed.
*/
function clearPagesIndex() {
return fs.unlink(shortIndexFile)
.then(() => {
Expand All @@ -139,7 +162,9 @@ function clearPagesIndex() {
});
}

// Set the shortIndex variable to null.
/**
* Set the shortIndex variable to null.
*/
function clearRuntimeIndex() {
shortIndex = null;
}
Expand All @@ -150,18 +175,26 @@ function rebuildPagesIndex() {
});
}

// If the variable is not set, read the file and set it.
// Else, just return the variable.
/**
* Return the index, that contains all available commands with their target os
* and platform. If the index is not loaded, read the file and load it.
*
* @returns {Promise<any>} The index entries.
*/
function getShortIndex() {
if (shortIndex) {
return Promise.resolve(shortIndex);
}
return readShortPagesIndex();
}

// Read the index file, and load it into memory.
// If the file does not exist, create the data structure, write the file,
// and load it into memory.
/**
* Read the index file, and load it into memory.
*
* If the file does not exist, create the data structure, write the file,
* and load it into memory.
* @returns {Promise<any>} The index entries.
*/
function readShortPagesIndex() {
return fs.readJson(shortIndexFile)
.then((idx) => {
Expand Down
1 change: 1 addition & 0 deletions lib/parser.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ function unhtml(text){

exports.parse = (markdown) => {
// Creating the page structure
/** @type {Required<import('./tldr').TldrPage> & { examples: any[] }} */
let page = {
name: '',
description: '',
Expand Down
13 changes: 9 additions & 4 deletions lib/remote.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,12 @@ const unzip = require('adm-zip');
const config = require('./config');
const axios = require('axios');

// Downloads the zip file from github and extracts it to folder
/**
* Download the zip file from GitHub and extract it to folder.
* @param {string} loc Path to a directory on disk.
* @param {string} lang Language/locale code.
* @returns {Promise<void>} A promise when the operation is completed.
*/
exports.download = (loc, lang) => {
// If the lang is english then keep the url simple, otherwise add language.
const suffix = (lang === 'en' ? '' : '.' + lang);
Expand All @@ -21,12 +26,12 @@ exports.download = (loc, lang) => {
headers: { 'User-Agent' : 'tldr-node-client' },
timeout: REQUEST_TIMEOUT,
}).then((response) => {
return new Promise((resolve, reject) => {
return new Promise((/** @type {(v?: never) => void} */ resolve, reject) => {
let fileName = path.join(loc, 'download_' + lang + '.zip');

const writer = fs.createWriteStream(fileName);
response.data.pipe(writer);

writer.on('finish', () => {
writer.end();
const zip = new unzip(fileName);
Expand All @@ -41,4 +46,4 @@ exports.download = (loc, lang) => {
}).catch((err) => {
return Promise.reject(err);
});
};
};
10 changes: 8 additions & 2 deletions lib/render.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,14 @@
const Theme = require('./theme');
const he = require('he'); // Import the 'he' library

// The page structure is passed to this function, and then the theme is applied
// to different parts of the page and rendered to the console
/**
* Page structure is passed to this function, and then the theme is applied to
* different parts of the page and rendered to the console.
*
* @param {import('./tldr').TldrPage} page
* @param {any} config
* @returns {string|void}
*/
exports.toANSI = (page, config) => {
// Creating the theme object
let themeOptions = config.themes[config.theme];
Expand Down
69 changes: 51 additions & 18 deletions lib/search.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,26 +9,53 @@ const utils = require('./utils');
const index = require('./index');
const platforms = require('./platforms');

/**
* @typedef {object} Corpus
* @property {Record<string, Record<string, number>>} fileWords
* @property {Record<string, number>} fileLengths
* @property {Record<string, string[]>} invertedIndex
* @property {any} allTokens
* @property {Record<string, Record<string, number>>} tfidf
*
* @typedef {object} Query
* @property {?string} raw
* @property {?string[]} tokens
* @property {Record<string, number>} frequency
* @property {Record<string, number>} score
* @property {QueryRank[]} ranks
*
* @typedef {object} QueryRank
* @property {string} file
* @property {number} score
*/

const CACHE_FOLDER = path.join(config.get().cache, 'cache');

const filepath = CACHE_FOLDER + '/search-corpus.json';

let corpus = {};

corpus.fileWords = {};
corpus.fileLengths = {};
corpus.invertedIndex = {};
corpus.allTokens = new Set();
corpus.tfidf = {};

let query = {};
/** @type {Corpus} */
let corpus = {
fileWords: {},
fileLengths: {},
invertedIndex: {},
allTokens: new Set(),
tfidf: {},
};

query.raw = null;
query.tokens = null;
query.frequency = {};
query.score = {};
query.ranks = [];
/** @type {Query} */
let query = {
raw: null,
tokens: null,
frequency: {},
score: {},
ranks: [],
};

/**
* @param {string} data
* @returns {string[]}
*/
let getTokens = (data) => {
let tokenizer = new natural.WordTokenizer();
let tokens = tokenizer.tokenize(data);
Expand Down Expand Up @@ -139,6 +166,9 @@ let readCorpus = () => {
});
};

/**
* @param {string} rawquery
*/
let processQuery = (rawquery) => {
query.raw = rawquery;
query.tokens = getTokens(rawquery);
Expand All @@ -156,10 +186,9 @@ let processQuery = (rawquery) => {
query.score = {};
query.tokens.forEach((word) => {
if (corpus.invertedIndex[word]) {
let logbase = 10;
let df = corpus.invertedIndex[word].length;
let idf = Math.log(numberOfFiles / df, logbase);
let wordWeight = idf * (1 + Math.log(query.frequency[word], logbase));
let idf = Math.log10(numberOfFiles / df);
let wordWeight = idf * (1 + Math.log10(query.frequency[word]));
corpus.invertedIndex[word].forEach((file) => {
let fileWeight = corpus.tfidf[file][word];
if (query.score[file]) {
Expand Down Expand Up @@ -209,7 +238,7 @@ exports.printResults = (results, config) => {
outputs.add(output);
});

console.log('Searching for:', query.raw.trim());
console.log('Searching for:', /** @type {string} */ (query.raw).trim());
console.log();
Array.from(outputs).forEach((elem) => {
console.log(elem);
Expand All @@ -220,7 +249,7 @@ exports.printResults = (results, config) => {
};

exports.createIndex = () => {
return utils.glob(CACHE_FOLDER + '/pages/**/*.md', {})
return utils.glob(CACHE_FOLDER + '/pages/**/*.md')
.then((files) => {
let promises = [];
files.forEach((file) => {
Expand All @@ -246,6 +275,10 @@ exports.createIndex = () => {
});
};

/**
* @param {string} rawquery
* @returns
*/
exports.getResults = (rawquery) => {
query.ranks = [];
return readCorpus()
Expand Down
Loading