fix(core): change api "older" rules

.teamcity/builds/apiReferences/BuildApiPages.kt

@@ -136,10 +136,11 @@ fun scriptGenerateSitemap(pagesRoot: String = DEFAULT_DOKKA_PATH): BuildStep = s
 fun scriptNoRobots(pagesRoot: String, block: ScriptBuildStep.() -> Unit = {}) = ScriptBuildStep {
     name = "Add no robots for older versions"
     workingDir = pagesRoot
-    //language=bash
+    //language=sh
     scriptContent = """
-        #!/bin/sh
-        find . -type f -path "*/api/*/older/*.html" -exec sed -i -E 's/(<head[^>]*>)/\1<meta name="robots" content="noindex, nofollow">/g' {} \;
+        # replace with print
+        # find . -type f \( -regex "^./\d\+\.\d\+\(\.\d\)\?/.\+\.html${'$'}" \) -exec echo {} \; -exec sed -i -E 's/(<head[^>]*>)/\1<meta name="robots" content="noindex, nofollow">/g' {} \;
+        find . -type f \( -regex "^./\d\+\.\d\+\(\.\d\)\?/.\+\.html${'$'}" \) -exec sed -i -E 's/(<head[^>]*>)/\1<meta name="robots" content="noindex, nofollow">/g' {} \;
     """.trimIndent()
     dockerImage = "alpine"
 }.apply(block)

robots.txt

@@ -3,5 +3,3 @@ Host: kotlinlang.org
 
 # Sitemaps
 sitemap: https://kotlinlang.org/sitemap_index.xml
-
-Disallow: /api/core/older/*

scripts/doindex/analyzer/index.ts

@@ -1,20 +1,21 @@
 import { env } from 'node:process';
 import { mkdir } from 'node:fs/promises';
-import { EventEmitter } from 'events'
+import { EventEmitter } from 'events';
 
 import { DIST_FOLDER, REPORT_FOLDER } from '../lib/files/index.js';
 import { execFilesTask, Filter } from '../lib/files/execTask.js';
 import { writeReports } from './reports/write.js';
 import { Metadata } from './metadata.js';
+import { isApiPreviousVersion } from '../lib/files/type.js';
 
 console.time('Data successfully built');
 
 const TASK_PATH = import.meta.dirname + '/task';
 
 function preFilterFiles({ relativePath: path }: Parameters<Filter>[0]) {
     const isSkip = (
-        // optimize by path "api/core/older" takes more than 1 minute only for filesystem iteration
-        (path.startsWith('api/') && (path.endsWith('/older'))) ||
+        // optimize by path "api/core/1.2.3/" takes more than 1 minute only for filesystem iteration
+        isApiPreviousVersion(path) ||
         path === 'spec' ||
         path === 'api/latest'
     );
@@ -25,7 +26,7 @@ function preFilterFiles({ relativePath: path }: Parameters<Filter>[0]) {
 
 let result: Metadata[] = [];
 
-EventEmitter.defaultMaxListeners = 15
+EventEmitter.defaultMaxListeners = 15;
 
 await execFilesTask(
     DIST_FOLDER, TASK_PATH,

scripts/doindex/lib/files/type.ts

@@ -10,6 +10,10 @@ export type FileType =
     | 'Page_Community' | 'Page_Documentation' | 'Page_LegacyDocumentation'
     | 'Iframe' | 'Redirect' | 'Hidden' | 'Unknown';
 
+export function isApiPreviousVersion(url: string) {
+    return url.match(/^api\/[^/]+\/(\d+\.\d+(\.\d+)?)\//)?.[1];
+}
+
 export async function getType(url: string, filePath: string): Promise<[FileType, CheerioAPI | null]> {
     if (url.endsWith('/package-list') || url.endsWith('index.yml')) return ['File_Text', null];
     if (url.endsWith('.pdf')) return ['File_Pdf', null];
@@ -47,7 +51,7 @@ export async function getType(url: string, filePath: string): Promise<[FileType,
             if (url.startsWith('api/latest/')) pageType = url.includes('jvm/stdlib') ? 'Page_API_stdlib' : 'Page_API_test';
             else if (url.endsWith('/navigation.html') && $(':is(.sideMenu, .sideMenuPart) .sideMenuPart').length)
                 pageType = 'Iframe';
-            else if (url.includes('/older/')) return ['Page_API_Deprecated', $];
+            else if (isApiPreviousVersion(url)) return ['Page_API_Deprecated', $];
             else pageType = 'Page_API';
         }