Merge branch 'master' into rtl

Sefaria · Apr 16, 2024 · 638fcd1 · 638fcd1
2 parents b5e6172 + e7965d2
commit 638fcd1
Show file tree

Hide file tree

Showing 72 changed files with 15,066 additions and 820 deletions.
diff --git a/.github/workflows/integration-testing.yaml b/.github/workflows/integration-testing.yaml
@@ -154,6 +154,7 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
   run-tests:
+    name: "Playwright"   # This name is referenced when slacking status
     needs:
       - build-derived
       - variables
@@ -188,7 +189,48 @@ jobs:
       - name: Wait for env
         run: while [[ $( curl -s -o /dev/null -w "%{http_code}" "nginx-integration-${{ needs.variables.outputs.commit }}.${{secrets.DEV_SANDBOX_NAMESPACE}}/texts") != "200" ]]; do sleep 10; done
         timeout-minutes: 5
-      - name: Run tests
-        run: if [[ $( curl -s -o /dev/null -w "%{http_code}" "nginx-integration-${{ needs.variables.outputs.commit }}.${{secrets.DEV_SANDBOX_NAMESPACE}}/texts") != "200" ]]; then exit 1; fi
+
+      - name: Set SANDBOX_URL
+        run: echo "SANDBOX_URL=http://nginx-integration-${{ needs.variables.outputs.commit }}.${{ secrets.DEV_SANDBOX_NAMESPACE }}/texts" >> $GITHUB_ENV
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps
+
+      - name: Run Playwright tests
+        run: npx playwright test
+
+      - name: Upload Playwright report
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
       - name: Uninstall
         run: helm delete integration-${{ needs.variables.outputs.commit }} -n ${{ secrets.DEV_SANDBOX_NAMESPACE }} --debug --timeout 10m0s
+  ending-notification:
+          runs-on: ubuntu-latest
+          needs:
+            - run-tests
+          steps:
+            - uses: actions/checkout@v2
+            - uses: actions/setup-node@v2
+              with:
+                node-version: '14'
+            - run: npm ci
+              working-directory: ./build/notify
+            - run: node notifyEnd.js
+              working-directory: ./build/notify
+              env:
+                SLACK_TEST_SUCCESS_WEBHOOK_URL: ${{secrets.SLACK_TEST_SUCCESS_WEBHOOK_URL}}
+                SLACK_TEST_FAILURE_WEBHOOK_URL: ${{secrets.SLACK_TEST_FAILURE_WEBHOOK_URL}}
+                GITUSER_SLACK_MAP: ${{secrets.GITUSER_SLACK_MAP}}
+                GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/production-deploy.yaml b/.github/workflows/production-deploy.yaml
@@ -55,7 +55,7 @@ jobs:
         uses: docker/setup-buildx-action@v1
       - id: auth
         name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v1
+        uses: google-github-actions/auth@v2
         with:
           token_format: 'access_token'
           workload_identity_provider: 'projects/${{ secrets.PROD_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
@@ -123,7 +123,7 @@ jobs:
         uses: docker/setup-buildx-action@v1
       - id: auth
         name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v1
+        uses: google-github-actions/auth@v2
         with:
           token_format: 'access_token'
           workload_identity_provider: 'projects/${{ secrets.PROD_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
@@ -185,18 +185,18 @@ jobs:
           ref: ${{ needs.workflow-check.outputs.ref }}
       - id: auth
         name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v1
+        uses: google-github-actions/auth@v2
         with:
           token_format: 'access_token'
           workload_identity_provider: 'projects/${{ secrets.PROD_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
           service_account: '${{ secrets.PROD_GKE_SA }}'
       - name: Setup GCloud
-        uses: google-github-actions/setup-gcloud@v0
+        uses: google-github-actions/setup-gcloud@v2
         with:
           project_id: ${{ secrets.PROD_GKE_PROJECT }}
           install_components: 'gke-gcloud-auth-plugin'
       - name: Authenticate GHA Runner To Target Cluster
-        uses: google-github-actions/get-gke-credentials@v0
+        uses: google-github-actions/get-gke-credentials@v2
         with:
           cluster_name: ${{secrets.PROD_GKE_CLUSTER}}
           location: ${{secrets.PROD_GKE_REGION}}

diff --git a/.github/workflows/readme-openapi-docs-sync.yml b/.github/workflows/readme-openapi-docs-sync.yml
@@ -0,0 +1,35 @@
+# This GitHub Actions workflow was auto-generated by the `rdme` cli on 2024-03-28T13:15:31.687Z
+# You can view our full documentation here: https://docs.readme.com/docs/rdme
+name: Publish API Docs
+
+on:
+  push:
+    branches:
+      # This workflow will run every time you push code to the following branch: `master`
+      # Check out GitHub's docs for more info on configuring this:
+      # https://docs.github.com/actions/using-workflows/events-that-trigger-workflows
+      - master
+    paths:
+      - 'docs/openAPI.json'
+
+jobs:
+  rdme-openapi:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v3
+
+      - name: Run validation on docs
+        uses: readmeio/rdme@v8
+        with:
+          rdme: openapi:validate docs/openAPI.json 
+
+      - name: Run `openapi` command (dry run)
+        uses: readmeio/rdme@v8
+        with:
+          rdme: openapi docs/openAPI.json --key=${{ secrets.README_API_KEY }} --id=${{ secrets.README_API_DEFINITIONS_ID }} --dryRun
+
+      - name: Run `openapi` command
+        uses: readmeio/rdme@v8
+        with:
+          rdme: openapi docs/openAPI.json --key=${{ secrets.README_API_KEY }} --id=${{ secrets.README_API_DEFINITIONS_ID }}
diff --git a/.gitignore b/.gitignore
@@ -133,3 +133,9 @@ static/sheetrank.json
 
 env/*
 dump/*
+/test-results/
+/playwright-report/
+/playwright/.cache/
+/test-results/
+/playwright-report/
+/playwright/.cache/
diff --git a/build/ci/production-values.yaml b/build/ci/production-values.yaml
@@ -98,6 +98,30 @@ varnish:
       ref: varnish-secret-production
   monitoring:
     enabled: true
+tasks:
+  enabled: true
+  resources:
+    requests:
+      memory: "1Gi"
+      cpu: "500m"
+    limits:
+      memory: "6Gi"
+      cpu: "1000m"
+  replicaCount: 1
+  redis:
+    url: ""
+    port: "26379"
+    brokerDBNumber: 2
+    resultBackendDBNumber: 3
+    sentinelURL: "redis-headless.redis.svc.cluster.local"
+    transportOptions:
+      master_name: mymaster
+    redisPassword:
+      ref: redis-password
+    sentinelPassword:
+      ref: sentinel-password
+  queues:
+    llm: llm-llm
 ingress:
   hosts:
     - host: "sefaria.org"
@@ -141,7 +165,10 @@ nginx:
   containerImage:
     imageRegistry:
     tag:
-  SEARCH_HOST: elasticsearch-es-http.elasticsearch.svc
+  SEARCH_HOST: "searchfix.cauldron.sefaria.org"
+  SEARCH_PORT: "443"
+  SEARCH_PATH: "api/search"
+  SEARCH_SSL_ENABLE: true
   disableScraping: false
   replicaCount: 2
   resources:
@@ -170,11 +197,11 @@ cronJobs:
   dailyEmailNotifications:
     enabled: true
   indexFromQueue:
-    enabled: true
+    enabled: false
   metrics:
     enabled: true
   nationBuilderSync:
-    enabled: true
+    enabled: false
   rambiWebpagesWeekly:
     enabled: true
   reindexElasticSearch:

diff --git a/build/linker/Dockerfile b/build/linker/Dockerfile
@@ -49,5 +49,5 @@ ENV PATH=/usr/local/nvidia/bin:$PATH
 ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64/
 ENV LD_PRELOAD=/usr/local/nvidia/lib64/libnvidia-ml.so
 
-RUN pip install spacy[cuda116]==3.4.1
+RUN pip install spacy[cuda116]~=3.7.0
 
diff --git a/build/notify/notifyEnd.js b/build/notify/notifyEnd.js
@@ -22,8 +22,11 @@ console.log(`
 const jobKeys = [
     "Jest",
     "PyTest",
+    "Playwright",
 ];
 
+const activeJobKeys = [];
+
 (async function() {
     // Get the committer via the commit
     let commit;
@@ -65,6 +68,7 @@ const jobKeys = [
     actionJobs.data.jobs.forEach(j => {
         if (jobKeys.includes(j.name)) {
             jobsResults[j.name] = {conclusion: j.conclusion, url: j.html_url};
+            activeJobKeys.push(j.name);
         }
     });
 
@@ -105,7 +109,7 @@ const jobKeys = [
 
     console.log(JSON.stringify(slackMsg));
 
-    const overallSuccess = jobKeys.every(k => succeeded(jobsResults[k]));
+    const overallSuccess = activeJobKeys.every(k => succeeded(jobsResults[k]));
     const webhookUrl = overallSuccess ? slackSuccessUrl : slackFailureUrl;
     const webhook = new IncomingWebhook(webhookUrl);