fix: copy specs local file refs (#2046)

## Proposed change

When we copy the spec file before generating the SDK we are breaking the
local relative paths referenced in the file.
The proposed change is to copy all the referenced files inside the SDK
in a folder `./spec-local-references/` to have them versioned.
We also update the local references in the spec to look for these files.

## Related issues

- 🐛 Fixes #2014
- 🚀 Feature #(issue)

<!-- Please make sure to follow the contributing guidelines on
https://github.com/amadeus-digital/Otter/blob/main/CONTRIBUTING.md -->

packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.spec.ts

@@ -0,0 +1,53 @@
+import { cleanVirtualFileSystem, useVirtualFileSystem } from '@o3r/test-helpers';
+import { readFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+
+describe('Specs processing', () => {
+  const virtualFileSystem = useVirtualFileSystem();
+  const {copyReferencedFiles, updateLocalRelativeRefs} = require('./copy-referenced-files');
+
+  const specsMocksPath = join(__dirname, '../../../../testing/mocks');
+  const specFilePath = '../models/split-spec/split-spec.yaml';
+  const outputDirectory = './local-references';
+
+  const copyMockFile = async (virtualPath: string, realPath: string) => {
+    if (!virtualFileSystem.existsSync(dirname(virtualPath))) {
+      await virtualFileSystem.promises.mkdir(dirname(virtualPath), {recursive: true});
+    }
+    await virtualFileSystem.promises.writeFile(virtualPath, await readFile(join(specsMocksPath, realPath), {encoding: 'utf8'}));
+  };
+
+  beforeAll(async () => {
+    await virtualFileSystem.promises.mkdir(dirname(specFilePath), {recursive: true});
+    await copyMockFile(specFilePath, 'split-spec/split-spec.yaml');
+    await copyMockFile('../models/split-spec/spec-chunk1.yaml', 'split-spec/spec-chunk1.yaml');
+    await copyMockFile('../models/spec-chunk2.yaml', 'spec-chunk2.yaml');
+    await copyMockFile('../models/spec-chunk3/spec-chunk3.yaml', 'spec-chunk3/spec-chunk3.yaml');
+    await copyMockFile('../models/spec-chunk4/spec-chunk4.yaml', 'spec-chunk4/spec-chunk4.yaml');
+  });
+
+  afterAll(() => {
+    cleanVirtualFileSystem();
+  });
+
+  it('should copy the local files referenced in the spec', async () => {
+    const baseRelativePath = await copyReferencedFiles(specFilePath, outputDirectory);
+    expect(baseRelativePath).toMatch(/^local-references[\\/]split-spec$/);
+    expect(virtualFileSystem.existsSync(join(outputDirectory, 'split-spec/split-spec.yaml'))).toBe(true);
+    expect(virtualFileSystem.existsSync(join(outputDirectory, 'split-spec/spec-chunk1.yaml'))).toBe(true);
+    expect(virtualFileSystem.existsSync(join(outputDirectory, 'spec-chunk2.yaml'))).toBe(true);
+    expect(virtualFileSystem.existsSync(join(outputDirectory, 'spec-chunk3/spec-chunk3.yaml'))).toBe(true);
+    expect(virtualFileSystem.existsSync(join(outputDirectory, 'spec-chunk4/spec-chunk4.yaml'))).toBe(true);
+  });
+
+  it('should update with new local basepath', async () => {
+    const specWitheRelativesFilePath = 'split-spec/split-spec.yaml';
+    const expectedSpecWitheRelativesFilePath = 'split-spec/spec-with-updated-paths.yaml';
+    const expectedContent = await readFile(join(specsMocksPath, expectedSpecWitheRelativesFilePath), {encoding: 'utf8'});
+    const specContent = await readFile(join(specsMocksPath, specWitheRelativesFilePath), {encoding: 'utf8'});
+
+    const baseRelativePath = await copyReferencedFiles(specFilePath, './output-local-directory');
+    const newSpecContent = await updateLocalRelativeRefs(specContent, baseRelativePath);
+    expect(newSpecContent).toBe(expectedContent);
+  });
+});

packages/@ama-sdk/schematics/schematics/typescript/core/helpers/copy-referenced-files.ts

@@ -0,0 +1,93 @@
+import { existsSync } from 'node:fs';
+import { copyFile, mkdir, readFile, rm } from 'node:fs/promises';
+import { dirname, join, normalize, posix, relative, resolve, sep } from 'node:path';
+
+const refMatcher = /\B['"]?[$]ref['"]?\s*:\s*([^#\n]+)/g;
+
+/**
+ * Extract the list of local references from a single spec file content
+ * @param specContent
+ * @param basePath
+ */
+function extractRefPaths(specContent: string, basePath: string): string[] {
+  const refs = specContent.match(refMatcher);
+  return refs ?
+    refs
+      .map((capture) => capture.replace(refMatcher, '$1').replace(/['"]/g, ''))
+      .filter((refPath) => refPath.startsWith('.'))
+      .map((refPath) => join(basePath, refPath))
+    : [];
+}
+
+/**
+ * Recursively extract the list of local references starting from the input spec file
+ * @param specFilePath
+ * @param referenceFilePath
+ * @param visited
+ */
+async function extractRefPathRecursive(specFilePath: string, referenceFilePath: string, visited: Set<string>): Promise<string[]> {
+  const resolvedFilePath = resolve(specFilePath);
+  if (!visited.has(resolvedFilePath)) {
+    visited.add(resolvedFilePath);
+
+    const specContent = await readFile(specFilePath, {encoding: 'utf8'});
+    const refPaths = extractRefPaths(specContent, relative(dirname(referenceFilePath), dirname(specFilePath)));
+    const recursiveRefPaths = await Promise.all(
+      refPaths.map((refPath) => extractRefPathRecursive(join(dirname(referenceFilePath), refPath), referenceFilePath, visited))
+    );
+    return [
+      ...refPaths,
+      ...recursiveRefPaths.flat()
+    ];
+  }
+  return [];
+}
+
+/**
+ * Replace all the local relative references using the new base relative path
+ * @param specContent
+ * @param newBaseRelativePath
+ */
+export function updateLocalRelativeRefs(specContent: string, newBaseRelativePath: string) {
+  const formatPath = (inputPath:string) => (inputPath.startsWith('.') ? inputPath : `./${inputPath}`).replace(/\\+/g, '/');
+  return specContent.replace(refMatcher, (match, ref: string) => {
+    const refPath = ref.replace(/['"]/g, '');
+    return refPath.startsWith('.') ?
+      match.replace(refPath, formatPath(normalize(posix.join(newBaseRelativePath.replaceAll(sep, posix.sep), refPath))))
+      : match;
+  });
+}
+
+/**
+ * Copy the local files referenced in the input spec file to the output directory
+ * @param specFilePath
+ * @param outputDirectory
+ */
+export async function copyReferencedFiles(specFilePath: string, outputDirectory: string) {
+  const dedupe = (paths: string[]) => ([...new Set(paths)]);
+  const allRefPaths = await extractRefPathRecursive(specFilePath, specFilePath, new Set());
+  const refPaths = dedupe(allRefPaths);
+  if (refPaths.length) {
+    if (existsSync(outputDirectory)) {
+      await rm(outputDirectory, { recursive: true });
+    }
+
+    // Calculate the lowest level base path to keep the same directory structure
+    const maxDepth = Math.max(...refPaths.map((refPath) => refPath.split('..').length));
+    const basePath = join(specFilePath, '../'.repeat(maxDepth));
+    const baseRelativePath = relative(basePath, dirname(specFilePath));
+
+    // Copy the files
+    await Promise.all(refPaths.map(async (refPath) => {
+      const sourcePath = join(dirname(specFilePath), refPath);
+      const destPath = join(outputDirectory, baseRelativePath, refPath);
+      if (!existsSync(dirname(destPath))) {
+        await mkdir(dirname(destPath), { recursive: true });
+      }
+      await copyFile(sourcePath, destPath);
+    }));
+
+    return join(outputDirectory, baseRelativePath);
+  }
+  return '';
+}

packages/@ama-sdk/schematics/schematics/typescript/core/index.ts

@@ -24,6 +24,7 @@ import { OpenApiCliOptions } from '../../code-generator/open-api-cli-generator/o
 import { treeGlob } from '../../helpers/tree-glob';
 import { NgGenerateTypescriptSDKCoreSchematicsSchema } from './schema';
 import { OpenApiCliGenerator } from '../../code-generator/open-api-cli-generator/open-api-cli.generator';
+import { copyReferencedFiles, updateLocalRelativeRefs } from './helpers/copy-referenced-files';
 import { generateOperationFinderFromSingleFile } from './helpers/path-extractor';
 
 const JAVA_OPTIONS = ['specPath', 'specConfigPath', 'globalProperty', 'outputPath'];
@@ -153,10 +154,20 @@ function ngGenerateTypescriptSDKFn(options: NgGenerateTypescriptSDKCoreSchematic
     let specContent!: string;
     if (URL.canParse(generatorOptions.specPath) && (new URL(generatorOptions.specPath)).protocol.startsWith('http')) {
       specContent = await (await fetch(generatorOptions.specPath)).text();
+      specContent = updateLocalRelativeRefs(specContent, path.dirname(generatorOptions.specPath));
     } else {
       const specPath = path.isAbsolute(generatorOptions.specPath) || !options.directory ?
         generatorOptions.specPath : path.join(options.directory, generatorOptions.specPath);
       specContent = readFileSync(specPath, {encoding: 'utf-8'}).toString();
+
+      if (path.relative(process.cwd(), specPath).startsWith('..')) {
+        // TODO would be better to create files on tree instead of FS
+        // https://github.com/AmadeusITGroup/otter/issues/2078
+        const newRelativePath = await copyReferencedFiles(specPath, './spec-local-references');
+        if (newRelativePath) {
+          specContent = updateLocalRelativeRefs(specContent, newRelativePath);
+        }
+      }
     }
 
     try {

packages/@ama-sdk/schematics/testing/mocks/spec-chunk2.yaml

@@ -0,0 +1,9 @@
+title: Pet
+type: object
+properties:
+  id:
+    type: integer
+    format: int64
+    example: 10
+  category:
+    $ref: './split-spec/split-spec.yaml#/components/schemas/Category'

packages/@ama-sdk/schematics/testing/mocks/spec-chunk3/spec-chunk3.yaml

@@ -0,0 +1,9 @@
+title: Pet
+type: object
+properties:
+  id:
+    type: integer
+    format: int64
+    example: 10
+  category:
+    $ref: '../spec-chunk4/spec-chunk4.yaml#/components/schemas/Category'

packages/@ama-sdk/schematics/testing/mocks/spec-chunk4/spec-chunk4.yaml

@@ -0,0 +1,12 @@
+components:
+  schemas:
+    Category:
+      type: object
+      properties:
+        id:
+          type: integer
+          format: int64
+          example: 10
+        name:
+          type: string
+          example: "test"

packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-chunk1.yaml

@@ -0,0 +1,11 @@
+title: Pet
+type: object
+properties:
+  id:
+    type: integer
+    format: int64
+    example: 10
+  category:
+    $ref: './split-spec.yaml#/components/schemas/Category'
+  category2:
+      $ref: '../spec-chunk4/spec-chunk4.yaml#/components/schemas/Category'

packages/@ama-sdk/schematics/testing/mocks/split-spec/spec-with-updated-paths.yaml

@@ -0,0 +1,45 @@
+openapi: 3.0.2
+info:
+  description: test
+  title: test
+  version: 0.0.0
+paths:
+  /test:
+    get:
+      responses:
+        '200':
+          description: test
+          content:
+            application/json:
+              schema:
+                $ref: './output-local-directory/split-spec/spec-chunk1.yaml'
+  /test2:
+    get:
+      responses:
+        '200':
+          description: test
+          content:
+            application/json:
+              schema:
+                $ref: './output-local-directory/spec-chunk2.yaml'
+  /test3:
+    get:
+      responses:
+        '200':
+          description: test
+          content:
+            application/json:
+              schema:
+                $ref: './output-local-directory/spec-chunk3/spec-chunk3.yaml'
+components:
+  schemas:
+    Category:
+      type: object
+      properties:
+        id:
+          type: integer
+          format: int64
+          example: 10
+        name:
+          type: string
+          example: "test"

packages/@ama-sdk/schematics/testing/mocks/split-spec/split-spec.yaml

@@ -0,0 +1,45 @@
+openapi: 3.0.2
+info:
+  description: test
+  title: test
+  version: 0.0.0
+paths:
+  /test:
+    get:
+      responses:
+        '200':
+          description: test
+          content:
+            application/json:
+              schema:
+                $ref: './spec-chunk1.yaml'
+  /test2:
+    get:
+      responses:
+        '200':
+          description: test
+          content:
+            application/json:
+              schema:
+                $ref: '../spec-chunk2.yaml'
+  /test3:
+    get:
+      responses:
+        '200':
+          description: test
+          content:
+            application/json:
+              schema:
+                $ref: '../spec-chunk3/spec-chunk3.yaml'
+components:
+  schemas:
+    Category:
+      type: object
+      properties:
+        id:
+          type: integer
+          format: int64
+          example: 10
+        name:
+          type: string
+          example: "test"