Make tuttle work with very active repos that are deployed into a subcollection

src/index.html

@@ -89,7 +89,7 @@ <h1><img class="tuttle" src="images/HPTuttle-1866.png"></img>
             <fx-submission id="fetch"
                            method="get"
                            replace="none"
-                           url="./git/{instance('default')/repos/repo[index('list')]/@collection}"
+                           url="./git/{instance('default')/repos/repo[index('list')]/@collection => encode-for-uri()}"
                            serialization="none">
 
                 <fx-action event="submit">
@@ -109,7 +109,7 @@ <h1><img class="tuttle" src="images/HPTuttle-1866.png"></img>
                            method="post"
                            replace="instance"
                            instance="response"
-                           url="./git/{instance('default')/repos/repo[index('list')]/@collection}"
+                           url="./git/{instance('default')/repos/repo[index('list')]/@collection => encode-for-uri()}"
                            serialization="none">
 
                 <fx-action event="submit-done">
@@ -122,7 +122,7 @@ <h1><img class="tuttle" src="images/HPTuttle-1866.png"></img>
             </fx-submission>
 
             <fx-submission id="update"
-                           url="./git/{instance('default')/repos/repo[index('list')]/@collection}/incremental"
+                           url="./git/{instance('default')/repos/repo[index('list')]/@collection => encode-for-uri()}/incremental"
                            replace="instance"
                            instance="response"
                            method="post">
@@ -138,7 +138,7 @@ <h1><img class="tuttle" src="images/HPTuttle-1866.png"></img>
                     <fx-send submission="config"></fx-send>
                 </fx-action>
                 <fx-action event="submit-error">
-                    <fx-message>Udpate Failed!</fx-message>
+                    <fx-message>Update Failed!</fx-message>
                     <fx-setvalue ref="instance('vars')/inprogress"></fx-setvalue>
                 </fx-action>
             </fx-submission>
@@ -213,4 +213,4 @@ <h3>Git Repositories</h3>
     <script type="module" src="js/fore-all.js"></script>
 </div>
 </body>
-</html>
+</html>

src/modules/api.xq

@@ -551,7 +551,7 @@ declare %private function api:get-collection-config($collection as xs:string?) a
         if (empty($git-collection))
         then error((), "git collection not found!")
         else if (empty($collection-config))
-        then error((), "collection config not found!")
+        then error((), "collection config " || $git-collection || " not found!")
         else $collection-config
 };
 

src/modules/github.xqm

@@ -9,6 +9,9 @@ import module namespace config="http://e-editiones.org/tuttle/config" at "config
 
 declare namespace http="http://expath.org/ns/http-client";
 
+declare variable $github:max-page-size := 100;
+declare variable $github:max-total-result-size := 500;
+
 declare function github:repo-url($config as map(*)) as xs:string {
     ``[`{$config?baseurl}`repos/`{$config?owner}`/`{$config?repo}`]``
 };
@@ -74,7 +77,7 @@ declare function github:get-last-commit($config as map(*)) as map(*) {
  : Get all commits
  :)
 declare function github:get-commits($config as map(*)) as array(*)* {
-    github:get-commits($config, 100)
+    github:get-commits($config, $github:max-page-size)
 };
 
 (:~
@@ -107,23 +110,50 @@ declare %private function github:short-commit-info ($commit-info as map(*)) as a
  : Get commits in full
  :)
 declare function github:get-raw-commits($config as map(*), $count as xs:integer) as array(*)? {
-    github:request-json-ignore-pages(
-        github:commit-ref-url($config, $count), $config?token)
+  github:get-raw-commits($config, $count, ())
+};
+
+(:~
+ : Get commits in full, going over pages until we find the commit with the correct hash
+ :)
+declare function github:get-raw-commits (
+  $config as map(*),
+  $count as xs:integer,
+  $stop-at-commit-id as xs:string?
+) as array(*)? {
+  let $stop-condition := if (empty($stop-at-commit-id)) then
+    function ($_) {
+      (: We are not looking for any SHA. Prevent going over all of the commits in a possibly big repo :)
+      true()
+    }
+  else
+    function ($results-on-page) {
+			(: We can stop searching once we have all the commits between 'now' and the commit we looked for :)
+      let $found-commits := $results-on-page?*?sha
+      return $found-commits = $stop-at-commit-id
+    }
+
+  let $results := github:request-json-all-pages(
+    github:commit-ref-url($config, $count),
+    $config?token,
+    $stop-condition
+  )
+  return array { $results?* }
 };
 
 (:~
  : Get diff between production collection and github-newest
  :)
 declare function github:get-newest-commits($config as map(*)) as xs:string* {
     let $deployed := $config?deployed
-    let $commits := github:get-raw-commits($config, 100)
+    let $commits := github:get-raw-commits($config, $github:max-page-size, $deployed)
     let $sha := $commits?*?sha
     let $how-many := index-of($sha, $deployed) - 1
     return
         if (empty($how-many)) then (
             error(
                 xs:QName("github:commit-not-found"),
-                'The deployed commit hash ' || $deployed || ' was not found in the list of commits on the remote.')
+                'The deployed commit hash ' || $deployed || ' was not found in the list of commits on the remote. Tuttle can only process incremental upgrades of ' || $github:max-total-result-size || '.')
         ) else (
             reverse(subsequence($sha, 1, $how-many))
         )
@@ -218,7 +248,7 @@ declare function github:incremental($config as map(*)) {
  :)
 declare function github:get-commit-files($config as map(*), $sha as xs:string) as array(*) {
     let $url := github:repo-url($config) || "/commits/"  || $sha
-    let $commit := github:request-json-all-pages($url, $config?token, ())
+    let $commit := github:request-json-all-pages($url, $config?token)
 
     return array { $commit?files?* }
 };
@@ -361,7 +391,32 @@ declare %private function github:request-json($url as xs:string, $token as xs:st
     )
 };
 
-declare %private function github:request-json-all-pages($url as xs:string, $token as xs:string?, $acc) {
+(:~
+ : Get all pages of a specified URL. Github has some paginated endpoints, This function traverses all of
+ : those and joins the results.
+ :)
+declare %private function github:request-json-all-pages($url as xs:string, $token as xs:string?) {
+  github:request-json-all-pages($url, $token, function ($_) { (: Traverse all pages :) false() }, ())
+};
+
+(:~
+ : Overload, adds the $stop-condition callback which is given the contents of the current page
+ : return `true()` to indicate there are sufficient results and we can stop
+ :)
+declare %private function github:request-json-all-pages(
+  $url as xs:string,
+  $token as xs:string?,
+  $stop-condition as function(map(*)) as xs:boolean
+) {
+  github:request-json-all-pages($url, $token, $stop-condition, ())
+};
+
+declare %private function github:request-json-all-pages(
+  $url as xs:string,
+  $token as xs:string?,
+  $stop-condition as function(map(*)) as xs:boolean,
+  $acc
+) {
     let $response :=
         app:request-json(
             github:build-request($url, (
@@ -374,13 +429,16 @@ declare %private function github:request-json-all-pages($url as xs:string, $toke
             github:parse-link-header($response[1]/http:header[@name="link"]/@value)?next
         ) else ()
 
-    let $all := ($acc, $response[2])
+    let $results-in-this-page := $response[2]
+    let $all := ($acc, $results-in-this-page)
+
+    let $should-stop := $stop-condition($results-in-this-page)
 
     return (
-        if (exists($next-url)) then (
-            github:request-json-all-pages($next-url, $token, $all)
+        if (not($should-stop) and count($all?*) < $github:max-total-result-size and exists($next-url)) then (
+           github:request-json-all-pages($next-url, $token, $stop-condition, $all)
         ) else (
-            $all
+          $all
         )
     )
 };

test/fixtures/alt-big-repo-tuttle.xml

@@ -0,0 +1,31 @@
+<tuttle>
+  <repos>
+    <!-- A version of tuttle sample data with tons of commits so updates go over multiple pages -->
+    <collection name="tuttle-sample-data">
+      <default>true</default>
+      <type>github</type>
+      <baseurl>https://api.github.com/</baseurl>
+      <repo>tuttle-sample-data</repo>
+      <owner>eeditiones</owner>
+      <token>XXX</token>
+      <ref>long-history</ref>
+      <hookuser>admin</hookuser>
+      <hookpasswd />
+    </collection>
+  </repos>
+  <ignore>
+    <file>existdb.json</file>
+    <file>build.xml</file>
+    <file>README.md</file>
+    <file>.gitignore</file>
+    <file>expath-pkg.xml.tmpl</file>
+    <file>repo.xml.tmpl</file>
+    <file>build.properties.xml</file>
+  </ignore>
+  <config
+    apikeys="/db/system/auth/tuttle-token.xml"
+    lock="git-lock.xml"
+    prefix="/db/apps/"
+    suffix="-stage"
+  ><sm group="nogroup" mode="rw-r--r--" user="nobody" /></config>
+</tuttle>

test/fixtures/alt-tuttle.xml

@@ -36,4 +36,4 @@
   <config prefix="/db/apps/" suffix="-stage" lock="git-lock.xml" apikeys="/db/system/auth/tuttle-token.xml">
     <sm user="nobody" group="nogroup" mode="rw-r--r--"/>
   </config>
-</tuttle>
+</tuttle>

test/tuttle.js

@@ -110,7 +110,7 @@ export default () =>
                 await assert.doesNotReject(resultPromise, 'The request should succeed');
                 res = await resultPromise;
             })
-            
+
             it('returns status 200', async function () {
                 assert.strictEqual(res.status, 200);
             });
@@ -176,27 +176,62 @@ export default () =>
         it('can also write hashes to repo.xml', async () => {
             await remove();
             await install();
-        
+
             // Set up tuttle with a repo where repo.xml is used to store the git sha info
             const buffer = await readFile('./test/fixtures/alt-repoxml-tuttle.xml');
             await putResource(buffer, '/db/apps/tuttle/data/tuttle.xml');
-        
+
             const resultPromise = axios.get('git/status', { auth });
             await assert.doesNotReject(resultPromise);
-        
+
             const stagingPromise = axios.get('git/tuttle-sample-data', { auth });
             await assert.doesNotReject(stagingPromise, 'The request should succeed');
 
             const deployPromise = axios.post('git/tuttle-sample-data', {}, { auth });
             await assert.doesNotReject(deployPromise, 'The request should succeed');
-        
+
             const repoXML = await getResource('/db/apps/tuttle-sample-data/repo.xml');
-        
+
             const repo = new DOMParser().parseFromString(repoXML.toString(), 'text/xml').documentElement;
             assert.ok(repo.getAttribute('commit-id'), 'The commit id should be set');
             assert.ok(repo.getAttribute('commit-time'), 'The commit time should be set');
             assert.ok(repo.getAttribute('commit-date'), 'The commit date should be set');
         });
-
-    });
 
+        describe('large histories', async () => {
+            before(async () => {
+                await remove();
+                await install();
+            });
+
+            await it('can upgrade over a few hundred commits', async () => {
+                // Set up tuttle with a repo with a ton of commits that it can upgrade over
+                const buffer = await readFile('./test/fixtures/alt-big-repo-tuttle.xml');
+                await putResource(buffer, '/db/apps/tuttle/data/tuttle.xml');
+
+                const OLD_HASH = '41188098f120b6e70d1b0c3bb704a422eba43dfa';
+                const stageOldVersionPromise = axios.get(`git/tuttle-sample-data?hash=${OLD_HASH}`, { auth });
+                await assert.doesNotReject(stageOldVersionPromise);
+                const deployOldVersionPromise = axios.post('git/tuttle-sample-data', {}, { auth });
+                await assert.doesNotReject(deployOldVersionPromise, 'The request should succeed');
+
+                const beforeString = await getResource('/db/apps/tuttle-sample-data/data/regular-changing-document.xml');
+
+                const before = new DOMParser().parseFromString(beforeString.toString(), 'text/xml').documentElement;
+                assert.strictEqual(before.textContent, 'Initial version');
+
+                console.log('deployed older version of the sample data on the long-history branch')
+
+                const resultPromise = axios.get('git/status', { auth });
+                await assert.doesNotReject(resultPromise);
+
+                const incrementalPromise = axios.post('git/tuttle-sample-data/incremental', {}, { auth });
+                await assert.doesNotReject(incrementalPromise, 'The incremental request should succeed');
+
+                const afterString = await getResource('/db/apps/tuttle-sample-data/data/regular-changing-document.xml');
+
+                const after = new DOMParser().parseFromString(afterString.toString(), 'text/xml').documentElement;
+                assert.strictEqual(after.textContent, 'change for 200');
+            });
+        });
+    });