fix: yarn berry + fetch + docker opti + sec (#1012)

* chore: wip yarn berry + fetch + docker opti + sec

* chore: wip

* chore: wip

* chore: wip

* chore: wip

* chore: wip

* chore: wip

* chore: wip

* chore: wip

* chore: wip

* chore: wip

* fix: docker cmd re-factorize

* fix: devdep

* fix: missing package for yarn start

* fix: types is used on runtime

* fix: add npmrc secret

* fix: npmTiptapToken

* fix: secret

* fix: var name

* fix: ci

* feat: enable buildkit service

* fix: wip

* fix: alert-cli package.json

* fix: eslint

* fix: jest

* fix: test ingester

* Update shared/elasticsearch-document-adapter/package.json

Co-authored-by: Martial Maillot <[email protected]>

* fix: private dila-resolver

* fix: up yarn.lock

* fix: docker compose build secret

---------

Co-authored-by: Martial Maillot <[email protected]>

.dockerignore

@@ -1,7 +1,13 @@
+**/.dockerignore
+**/Dockerfile
 .git
 .github
 .kontinuous
-*.md
+**/*.md
 **/node_modules
 **/.next/cache
 **/data/*
+shared/*/build
+targets/*/build
+shared/*/lib
+targets/*/lib

.envrc

@@ -0,0 +1 @@
+export NPM_TIPTAP_TOKEN=$(cat .npmTiptapToken.secret)

.github/workflows/quality.yml

@@ -22,12 +22,10 @@ jobs:
         with:
           node-version: 20.3.1
           cache: "yarn"
-      - name: Setup tiptap pro
-        run: |
-          echo "@tiptap-pro:registry=https://registry.tiptap.dev/" >> ~/.npmrc
-          echo "//registry.tiptap.dev/:_authToken=${{ secrets.TIPTAP_PRO_TOKEN }}" >> ~/.npmrc
       - name: Install dependencies
-        run: yarn install --prefer-offline --frozen-lockfile
+        env:
+          NPM_TIPTAP_TOKEN: ${{ secrets.TIPTAP_PRO_TOKEN }}
+        run: yarn --immutable
       - name: Build code
         run: |
           yarn build

.github/workflows/release.yml

@@ -23,7 +23,9 @@ jobs:
           node-version: 20.3.1
           cache: "yarn"
       - name: Install dependencies
-        run: yarn install --frozen-lockfile
+        env:
+          NPM_TIPTAP_TOKEN: ${{ secrets.TIPTAP_PRO_TOKEN }}
+        run: yarn --immutable
       - name: Set git
         run: |
           git config --global user.name "${NAME}"
@@ -39,12 +41,5 @@ jobs:
         run: GH_TOKEN=${GITHUB_TOKEN} yarn lerna version --force-publish --yes --conventional-commits --create-release github
         env:
           GITHUB_TOKEN: ${{ secrets.SOCIALGROOVYBOT_BOTO_PAT }}
-      - name: Setup token
-        run: echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} > .npmrc
-        env:
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
       - name: Deploy to npm
         run: yarn lerna publish from-package --yes
-      - name: Remove .npmrc
-        if: always()
-        run: rm .npmrc

.gitignore

@@ -5,4 +5,13 @@ docker-compose.override.yml
 .vscode
 *-error.log
 .swc
-.npmrc
+
+# Yarn Berry
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+
+.npmTiptapToken.secret

.kontinuous/config.yaml

@@ -1,7 +1,8 @@
 projectName: cdtn
 dependencies:
   fabrique:
-    import: socialgouv/kontinuous/plugins/fabrique
+    extends:
+    - name: buildkit-service
     dependencies:
       contrib:
         preDeploy:

.kontinuous/env/dev/templates/npm-tiptap.sealed-secret.yaml

@@ -0,0 +1,16 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  annotations:
+    sealedsecrets.bitnami.com/cluster-wide: 'true'
+  name: npm-tiptap
+  namespace: null
+spec:
+  encryptedData:
+    NPM_TIPTAP_TOKEN: AgBqJbVebE3DrTyTyinqeeSROcQlWEyMLHZ/mY+14sMAI+GWxTfOKsf+TGcdiM1CIzdhff/UCVotRJNHCYR19EH3X6R9/aRfQcOQ0gv2byI9Pv1MOqu/wNL2EG/OCYaMHHfDiUwKKbmtno8pXOmNR9MLxmCD+c3SZRSzV69zINtu6ESl3BtF9whcpQEpjV3oAesmFxNYiKMnIudFVShDQLa6ECTUk/9l9up92A74IIPTExjtWMlnbW/mbyqTzjurguA0GifUGYbr96hB8T6CWx3Ju5LxvzOlOPw4zrpy4Si1knzF4XLiU6Yahl3muNMVKGbOtmjN2sfLjSqA8NhBaHW0knau3TdMIhFTmUTqIDAfXFoM3oiwc4wiQLGoF29mbhEO2CBw9sGoeMGs5RGwm9HMa3yvkPUJn+UZd/uxDwTeptAqJYhNGMgqzDLn8yM8QtXQW9sI2kl0UdYZIyWnhZh7YBnDrqK7UBZU+IDer8pYeb1jPCeTHYQHBetadjI0Ps+aQIrfCc0DtNF0Uwoq+vo4RWi+WUDkNu4OisqB7+hMge4KjAqeGPGvS79O8k3egbQ7zEhRJU5ECn155Nl/DSYDdYtcOv43UHA7OABM/1BQmnmKOUabzNuUEKXGHrafASUyTNH09q+K9FWA+VvJ2ZHY+S2jX9mZc2yhsXbivsn/f30tog5WiY11/uHBLdvvOvfFpIR1Wah1X8tQbNBcQULLybDx2/IdwdBQVHxGgOmcrHn47r4dSHiDCmuBGg==
+  template:
+    metadata:
+      annotations:
+        sealedsecrets.bitnami.com/cluster-wide: 'true'
+      name: npm-tiptap
+    type: Opaque

.kontinuous/env/preprod/templates/npm-tiptap.sealed-secret.yaml

@@ -0,0 +1 @@
+../../dev/templates/npm-tiptap.sealed-secret.yaml

.kontinuous/env/prod/templates/npm-titap.sealed-secret.yaml

@@ -0,0 +1,16 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  annotations:
+    sealedsecrets.bitnami.com/namespace-wide: 'true'
+  name: npm-tiptap
+  namespace: cdtn-admin
+spec:
+  encryptedData:
+    NPM_TIPTAP_TOKEN: AgCYNRPICOxyRzp1TyuDWEhAfNrfseRqveDG5/2Q7PPnmmcPptoU/4GYCOeGf3mmXju96USLUoLGwhC3MN8FnW32kH547p0dvUXvwrrJVdxInjP9I8VXptQYPlfjjIFEs19VTj00H27DrapXEPlT4nAjAnHxeRlvtGoMR1xL3UrB2aq5v2j7w+6aY97HYqagrHYI1wkTE9vRRdzgP6wgJuI1ENswBIsVwL6LXRF/d1xxE2Rr6YANe4DXmdE7O+PNxoDzBIXw5B7ie9zNksJw8XbN22oUNzjL9AxS7Oha7XLyeLMc4/NDURhK/eqtBait/RFZNCREU6TPq/lZ90qDXZAm2CpMTk5mzNC6DftUjp9MPy+PurRoZ66HoPJocDZFEo2ZY3ERSSi8gFkwsg65mGpJBAajFgf4Ua0qH9RstlpE15O/+o42l3SvGhLnnkpjaFzz75KAvAkKIcGKMlOT9xl4Ek/HvGcKvzy3yvLCp1hlVJ6Wa/Mp3L9D8JcbXQMaCx9nXCxjEr0LHnucII5XXk4ANlddxVz6A6uWpcoz72zlfPQuOUhBp6jj0vzQQuCiQOs9w4xiuuudOGktG13vXKgMBLiW1qDxifS69fmwybQlpIc0udZ7GiothzyCT/++u8i1hJLbilmGWw1SvIYmoeYLyF3CT4Nd1p92Cz29HUbZmqs2HFORkTnHWTsM7ZbXDDeeMafyWZ0HfokZR8KO9efqa7I1oWMA1+UMXPLnieXD9AXeb1ShlxikGj5oxw==
+  template:
+    metadata:
+      annotations:
+        sealedsecrets.bitnami.com/namespace-wide: 'true'
+      name: npm-tiptap
+    type: Opaque

.kontinuous/templates/alert.job.tpl

@@ -4,6 +4,10 @@ spec:
   template:
     spec:
       restartPolicy: Never
+      securityContext:
+        fsGroup: 1000
+        runAsUser: 1000
+        runAsGroup: 1000
       containers:
         - name: update-alert
           image: "{{ or .Values.registry .Values.global.registry }}/{{ .Values.global.imageProject }}/{{ .Values.global.imageRepository }}/alert:{{ .Values.global.imageTag }}"

.kontinuous/templates/cleanup.job.tpl

@@ -2,6 +2,10 @@
 spec:
   template:
     spec:
+      securityContext:
+        fsGroup: 1000
+        runAsUser: 1000
+        runAsGroup: 1000
       containers:
         - name: db-cleaner
           image: ghcr.io/socialgouv/docker/psql:7.0.0

.kontinuous/templates/ingester.job.tpl

@@ -4,6 +4,10 @@ spec:
   template:
     spec:
       restartPolicy: Never
+      securityContext:
+        fsGroup: 1000
+        runAsUser: 1000
+        runAsGroup: 1000
       containers:
         - name: update-ingester
           image: "{{ or .Values.registry .Values.global.registry }}/{{ .Values.global.imageProject }}/{{ .Values.global.imageRepository }}/ingester:{{ .Values.global.imageTag }}"

.kontinuous/values.yaml

@@ -7,36 +7,36 @@ jobs:
         imagePackage: frontend
         dockerfile: targets/frontend/Dockerfile
         secrets:
-          npmrc:
-            secretName: npm
-            secretKey: NPMRC
+          npmTiptapToken:
+            secretName: npm-tiptap
+            secretKey: NPM_TIPTAP_TOKEN
     build-ingester:
       use: build
       with:
         imagePackage: ingester
         dockerfile: targets/ingester/Dockerfile
         secrets:
-          npmrc:
-            secretName: npm
-            secretKey: NPMRC
+          npmTiptapToken:
+            secretName: npm-tiptap
+            secretKey: NPM_TIPTAP_TOKEN
     build-alert:
       use: build
       with:
         imagePackage: alert
         dockerfile: targets/alert-cli/Dockerfile
         secrets:
-          npmrc:
-            secretName: npm
-            secretKey: NPMRC
+          npmTiptapToken:
+            secretName: npm-tiptap
+            secretKey: NPM_TIPTAP_TOKEN
     build-export:
       use: build
       with:
         imagePackage: export
         dockerfile: targets/export-elasticsearch/Dockerfile
         secrets:
-          npmrc:
-            secretName: npm
-            secretKey: NPMRC
+          npmTiptapToken:
+            secretName: npm-tiptap
+            secretKey: NPM_TIPTAP_TOKEN
     build-contributions:
       use: build
       with:
@@ -72,6 +72,10 @@ www:
       cpu: "50m"
       memory: "128Mi"
   replicas: 1
+  securityContext:
+    fsGroup: 1000
+    runAsUser: 1000
+    runAsGroup: 1000
 
 contributions:
   ~chart: app
@@ -92,6 +96,10 @@ contributions:
       cpu: "50m"
       memory: "128Mi"
   replicas: 1
+  securityContext:
+    fsGroup: 1000
+    runAsUser: 1000
+    runAsGroup: 1000
 
 export:
   ~chart: app
@@ -114,6 +122,10 @@ export:
       cpu: '50m'
       memory: 128Mi
   replicas: 1
+  securityContext:
+    fsGroup: 1000
+    runAsUser: 1000
+    runAsGroup: 1000
 
 hasura:
   ~chart: hasura
@@ -136,6 +148,10 @@ hasura:
     requests:
       cpu: '50m'
       memory: 650Mi
+  securityContext:
+    fsGroup: 1001
+    runAsUser: 1001
+    runAsGroup: 1001
 
 deactivate:
   jobs-deactivate:

.yarn/plugins/@yarnpkg/plugin-fetch.cjs

@@ -0,0 +1,19 @@
+/* eslint-disable */
+//prettier-ignore
+module.exports = {
+name: "@yarnpkg/plugin-fetch",
+factory: function (require) {
+var plugin=(()=>{var ie=Object.defineProperty;var le=(n,t,e)=>t in n?ie(n,t,{enumerable:!0,configurable:!0,writable:!0,value:e}):n[t]=e;var c=(n=>typeof require<"u"?require:typeof Proxy<"u"?new Proxy(n,{get:(t,e)=>(typeof require<"u"?require:t)[e]}):n)(function(n){if(typeof require<"u")return require.apply(this,arguments);throw new Error('Dynamic require of "'+n+'" is not supported')});var a=(n,t)=>()=>(t||n((t={exports:{}}).exports,t),t.exports);var u=(n,t,e)=>(le(n,typeof t!="symbol"?t+"":t,e),e);var $=a((_e,A)=>{"use strict";A.exports=function(t){return t.map(function(e){return e&&typeof e=="object"?e.op.replace(/(.)/g,"\\$1"):/["\s]/.test(e)&&!/'/.test(e)?"'"+e.replace(/(['\\])/g,"\\$1")+"'":/["'\s]/.test(e)?'"'+e.replace(/(["\\$`!])/g,"\\$1")+'"':String(e).replace(/([A-Za-z]:)?([#!"$&'()*,:;<=>?@[\\\]^`{|}])/g,"$1\\$2")}).join(" ")}});var v=a((ze,Y)=>{var C=c("fs"),pe=c("path"),{parseSyml:ue}=c("@yarnpkg/parsers"),B=["npm","portal","link"];Y.exports=function(){let t=C.readFileSync("yarn.lock","utf8"),e=ue(t),r=Object.keys(e).filter(o=>o.includes("@workspace:")),s=r.map(o=>{let[,i]=e[o].resolution.trim().split("@workspace:");return i==="."?null:i}).filter(Boolean);r.forEach(o=>{let{dependencies:i,dependenciesMeta:p,peerDependencies:b,peerDependenciesMeta:P,resolution:D,bin:oe}=e[o],[ce,w]=D.trim().split("@workspace:"),ae=pe.join(w,"package.json"),y={name:ce,version:"0.0.0",description:"**DON'T COMMIT** Generated file for caching",private:!0,dependencies:i,peerDependencies:b,peerDependenciesMeta:P,bin:oe};if(p){let h={};Object.keys(p).forEach(m=>{h[m]=i[m],delete i[m]}),y.optionalDependencies=h}if(w==="."){s.length>0&&(y.workspaces={packages:s});let h=Object.keys(e),m=l=>{let k=l.trim().split("@");return l.startsWith("@")?k=k.slice(0,2):k=k.slice(0,1),k.join("@")};y.resolutions=h.filter(l=>{if(l.includes("@workspace:")||l.includes(", ")||!B.some(f=>l.includes(`@${f}:`)))return!1;let k=m(l);return h.every(f=>l===f?!0:f.split(",").map(g=>m(g)).every(g=>g!==k))}).reduce((l,k)=>(B.forEach(f=>{if(!k.includes(`@${f}:`))return;let[g,x]=k.trim().split(`@${f}:`);switch(f){case"npm":l[g]=x.includes("@")?`${f}:${x}`:x;break;case"portal":case"link":l[g]=`${f}:${x.split("::")[0]}`;break}}),l),{})}C.mkdirSync(w,{recursive:!0}),C.writeFileSync(ae,`${JSON.stringify(y,null,2)}
+`)})}});var j=a((Ge,H)=>{var ke=v();H.exports=n=>{n.context.stdout.write(`[YARN-FETCH] extracting package.json file(s) from yarn.lock
+`),ke()}});var M=a((Ze,L)=>{var d=c("fs"),fe=c("path"),{execSync:de}=c("child_process"),{parseSyml:me}=c("@yarnpkg/parsers"),{BaseCommand:ge}=c("@yarnpkg/cli"),{Command:he,Option:R}=c("clipanion"),ye=$(),xe=j(),q;L.exports=(q=class extends ge{protectPackageJson=R.Boolean("--protect-package-json");args=R.Proxy();async execute(){let{protectPackageJson:t=process.stdout.isTTY}=this,e=[];if(t){this.context.stdout.write(`[YARN-FETCH] backup possible package.json file(s)
+`);let s=d.readFileSync("yarn.lock","utf8"),o=me(s);e=Object.keys(o).filter(p=>p.includes("@workspace:")).map(p=>{let{resolution:b}=o[p],[,P]=b.trim().split("@workspace:");return fe.join(P,"package.json")}),e.forEach(p=>{d.existsSync(p)&&!d.existsSync(`${p}.yarn-plugin-fetch-bak`)&&d.copyFileSync(p,`${p}.yarn-plugin-fetch-bak`)})}xe(this);let r=`yarn ${ye(this.args)}`;this.context.stdout.write(`[YARN-FETCH] ${r}
+`);try{de(r,{stdio:"inherit"})}catch(s){throw s}finally{t&&(this.context.stdout.write(`[YARN-FETCH] restoring possible package.json file(s)
+`),e.forEach(s=>{d.existsSync(`${s}.yarn-plugin-fetch-bak`)?d.renameSync(`${s}.yarn-plugin-fetch-bak`,s):d.unlinkSync(s)}))}}},u(q,"paths",[["fetch"]]),u(q,"usage",he.Usage({description:"fetch dependencies from yarn.lock in Docker build",details:`
+      expand yarn.lock to package.json file(s) and install dependencies in Docker build.
+    `,examples:[["yarn fetch --immutable","yarn fetch workspace my-package focus"]]})),q)});var K=a((Xe,I)=>{var{BaseCommand:qe}=c("@yarnpkg/cli"),be=j(),S;I.exports=(S=class extends qe{async execute(){be(this)}},u(S,"paths",[["fetch-tools","expand-lock"]]),S)});var _=a((tt,W)=>{function Pe(n,t,e){let r=t.split("."),s=n;for(let o of r){if(s[o]===void 0)return e;s=s[o]}return s}function we(n,t,e){let r=t.split("."),s=n;for(let o=0;o<r.length-1;o++){let i=r[o];(!s[i]||typeof s[i]!="object")&&(s[i]={}),s=s[i]}return s[r[r.length-1]]=e,n}function $e(n,t){let e=t.split("."),r=n;for(let s=0;s<e.length-1;s++){let o=e[s];if(!r[o])return!1;r=r[o]}return delete r[e[e.length-1]],!0}W.exports={get:Pe,set:we,unset:$e}});var F=a((st,G)=>{var z=c("fs"),{get:Ce,set:je,unset:Se}=_();G.exports=function(t,e){let r=JSON.parse(z.readFileSync("package.json","utf-8")),s=Ce(r,t);s!==void 0&&(je(r,e,s),Se(r,t),z.writeFileSync("package.json",JSON.stringify(r,null,2)))}});var E=a((nt,U)=>{var Fe=F();U.exports=function(){Fe("scripts._postinstall","scripts.postinstall")}});var Q=a((ot,Z)=>{var{BaseCommand:Ee}=c("@yarnpkg/cli"),Je=E(),J;Z.exports=(J=class extends Ee{async execute(){Je()}},u(J,"paths",[["fetch-tools","disable-postinstall"]]),J)});var N=a((at,V)=>{var Ne=F();V.exports=function(){Ne("scripts.postinstall","scripts._postinstall")}});var ee=a((lt,X)=>{var{BaseCommand:Te}=c("@yarnpkg/cli"),Oe=N(),T;X.exports=(T=class extends Te{async execute(){Oe()}},u(T,"paths",[["fetch-tools","disable-postinstall"]]),T)});var ne=a((kt,se)=>{var{execSync:De}=c("child_process"),{BaseCommand:Ae}=c("@yarnpkg/cli"),{Option:te}=c("clipanion"),Be=$(),Ye=E(),ve=N(),O;se.exports=(O=class extends Ae{postinstall=te.Boolean("--postinstall");args=te.Proxy();async execute(){this.postinstall||(this.context.stdout.write(`[YARN-FETCH] disable postinstall command in package.json
+`),ve());let t=`yarn workspaces focus --production ${Be(this.args)}`;this.context.stdout.write(`[YARN-FETCH] ${t}
+`),De(t,{stdio:"inherit"}),this.postinstall||(this.context.stdout.write(`[YARN-FETCH] re-enable postinstall command in package.json
+`),Ye())}},u(O,"paths",[["fetch-tools","production"]]),O)});var Ke=a((dt,re)=>{var He=M(),Re=K(),Le=Q(),Me=ee(),Ie=ne();re.exports={commands:[He,Re,Me,Le,Ie]}});return Ke();})();
+return plugin;
+}
+};