diff --git a/static/js/sefaria/sefaria.js b/static/js/sefaria/sefaria.js
index e310d85f25..4bbcf901a5 100644
--- a/static/js/sefaria/sefaria.js
+++ b/static/js/sefaria/sefaria.js
@@ -276,6 +276,12 @@ Sefaria = extend(Sefaria, {
let index = Sefaria.index(pRef.index);
return index && index.categories ? index.categories : [];
},
+ refIndexTitle: function(ref) {
+ let pRef = Sefaria.parseRef(ref);
+ if ("error" in pRef) { return null; }
+ let index = Sefaria.index(pRef.index);
+ return index?.title
+ },
sectionRef: function(ref, deriveIfNotFound=false) {
// Returns the section level ref for `ref` or null if no data is available
const oref = this.getRefFromCache(ref);
@@ -465,31 +471,64 @@ Sefaria = extend(Sefaria, {
});
},
_bulkTexts: {},
+ partitionArrayForURL: function(arr, urlMaxLength, dividerToken) {
+ const result = [];
+ const dividerTokenLength = encodeURIComponent(dividerToken).length;
+ let currentPartition = [];
+ let currentLength = 0;
+
+ for (let i = 0; i < arr.length; i++) {
+ // Calculate the length of the new item when added to the partition
+ const item = arr[i];
+ const encodedItem = encodeURIComponent(item);
+ const newLength = currentPartition.length === 0
+ ? encodedItem.length
+ : currentLength + encodedItem.length + dividerTokenLength; // consider dividerToken length
+
+ // Check if adding this item exceeds the max length
+ if (newLength > urlMaxLength) {
+ // If it does, push the current partition to the result and start a new one
+ result.push(currentPartition);
+ currentPartition = [];
+ currentLength = 0;
+ currentPartition.push(item);
+ continue
+ }
+
+ // Add the item to the current partition
+ currentPartition.push(item);
+ currentLength = newLength;
+ }
+
+ // Add the last partition to the result
+ if (currentPartition.length > 0) {
+ result.push(currentPartition);
+ }
+
+ return result;
+},
+
getBulkText: function(refs, asSizedString=false, minChar=null, maxChar=null, transLangPref=null) {
if (refs.length === 0) { return Promise.resolve({}); }
const MAX_URL_LENGTH = 3800;
- const hostStr = `${Sefaria.apiHost}/api/bulktext/`;
+ const ASSUMED_HOSTNAME_LENGTH_BOUND = 50;
+ const hostStr = encodeURI(`${Sefaria.apiHost}/api/bulktext/`);
let paramStr = '';
for (let [paramKey, paramVal] of Object.entries({asSizedString, minChar, maxChar, transLangPref})) {
paramStr = !!paramVal ? paramStr + `&${paramKey}=${paramVal}` : paramStr;
}
paramStr = paramStr.replace(/&/,'?');
+ paramStr = encodeURI(paramStr);
// Split into multiple requests if URL length goes above limit
- let refStrs = [""];
- refs.map(ref => {
- let last = refStrs[refStrs.length-1];
- const encodedFullURL = encodeURI(`${hostStr}${last}|${ref}${paramStr}`);
- if (encodedFullURL.length > MAX_URL_LENGTH) {
- refStrs.push(ref)
- } else {
- refStrs[refStrs.length-1] += last.length ? `|${ref}` : ref;
- }
- });
+ const limit = MAX_URL_LENGTH-(hostStr+paramStr).length-ASSUMED_HOSTNAME_LENGTH_BOUND
+ const refsSubArrays = this.partitionArrayForURL( refs, limit, '|');
+ const refStrs = refsSubArrays.map(refsSubArray => refsSubArray.join('|'));
+
let promises = refStrs.map(refStr => this._cachedApiPromise({
- url: encodeURI(`${hostStr}${refStr}${paramStr}`),
+ url: `${hostStr}${encodeURIComponent(refStr)}${paramStr}`,
key: refStr + paramStr,
store: this._bulkTexts
}));
diff --git a/static/js/sefaria/sheetsUtils.js b/static/js/sefaria/sheetsUtils.js
new file mode 100644
index 0000000000..850f9167b4
--- /dev/null
+++ b/static/js/sefaria/sheetsUtils.js
@@ -0,0 +1,69 @@
+import Sefaria from "./sefaria";
+
+export async function getSegmentObjs(refs) {
+ /*
+ Given an array of ref-strings (could also be ranged refs),
+ turn each ref to segment object and return an array of all segments
+ */
+ const segments = [];
+
+ for (const ref of refs) {
+ const text = await Sefaria.getText(ref, { stripItags: 1 });
+ const newSegments = Sefaria.makeSegments(text, false);
+ segments.push(...newSegments);
+ }
+ return segments;
+}
+export async function getNormalRef(ref) {
+ /*
+ Given a ref-string, get he and en normal ref-string
+ */
+ const refObj = await Sefaria.getRef(ref);
+ return {en: refObj.ref, he: refObj.heRef}
+}
+function placedSegmentMapper(lang, segmented, includeNumbers, s) {
+ /*
+ Map each segment object to a formatted text string
+ */
+ if (!s[lang]) {return ""}
+
+ let numStr = "";
+ if (includeNumbers) {
+ const num = (lang=="he") ? Sefaria.hebrew.encodeHebrewNumeral(s.number) : s.number;
+ numStr = "
(" + num + ") ";
+ }
+ let str = "
" + numStr + s[lang] + " ";
+ if (segmented) {
+ str = "
" + str + "
";
+ }
+ str = str.replace(/(
)+/g, ' ')
+ return str;
+}
+export const segmentsToSourceText = (segments, lan) => {
+ /*
+ Turn array of segment objects into one chunk of formatted text
+ */
+ const segmented = shouldBeSegmented(segments[0].ref);
+ const includeNumbers = shouldIncludeSegmentNums(segments[0].ref);
+ return(segments.map(placedSegmentMapper.bind(this, lan, segmented, includeNumbers))
+ .filter(Boolean)
+ .join(""));
+}
+function shouldIncludeSegmentNums(ref){
+ /*
+ Decide if segment of this ref should have segment numbers when turned into text chunk
+ */
+ const indexTitle = Sefaria.refIndexTitle(ref);
+ const categories = Sefaria.refCategories(ref);
+ if (categories.includes("Talmud")) {return false}
+ if (indexTitle === "Pesach Haggadah") {return false}
+ if (categories === 1) {return false}
+ return true;
+}
+function shouldBeSegmented(ref){
+ /*
+ Decide if segment of this ref should be followed by new line when turned into text chunk
+ */
+ const categories = Sefaria.refCategories(ref);
+ return !(categories[0] in {"Tanakh": 1, "Talmud": 1});
+}
\ No newline at end of file