feat(marketplace): Add marketplace providers to add entities into the catalog (#483)

* add marketplace providers to ingest the entities into catalog

* removed commented code

* unregister collection provider in 1.5

Author: karthikjeeyar

workspaces/marketplace/.changeset/shaggy-zoos-fetch.md

@@ -0,0 +1,5 @@
+---
+'@red-hat-developer-hub/backstage-plugin-catalog-backend-module-marketplace': patch
+---
+
+Add marketplace providers to add entities into catalog

workspaces/marketplace/package.json

@@ -11,7 +11,7 @@
     "start-backend": "yarn workspace backend start",
     "build:backend": "yarn workspace backend build",
     "tsc": "tsc",
-    "tsc:full": "tsc --skipLibCheck false --incremental false",
+    "tsc:full": "tsc --skipLibCheck true --incremental false",
     "build:all": "backstage-cli repo build --all",
     "build:api-reports": "yarn build:api-reports:only --tsc",
     "build:api-reports:only": "backstage-repo-tools api-reports -o ae-wrong-input-file-type,ae-undocumented --validate-release-tags",

workspaces/marketplace/plugins/catalog-backend-module-marketplace/package.json

@@ -39,11 +39,15 @@
     "@backstage/plugin-catalog-node": "^1.15.1",
     "@backstage/types": "^1.2.1",
     "@red-hat-developer-hub/backstage-plugin-marketplace-common": "workspace:^",
+    "find-root": "^1.1.0",
+    "glob": "^8.1.0",
+    "js-yaml": "^4.1.0",
     "semver": "^7.6.3"
   },
   "devDependencies": {
     "@backstage/backend-test-utils": "^1.2.1",
-    "@backstage/cli": "^0.29.5"
+    "@backstage/cli": "^0.29.5",
+    "@types/glob": "^8.1.0"
   },
   "files": [
     "dist"

workspaces/marketplace/plugins/catalog-backend-module-marketplace/report.api.md

@@ -11,9 +11,28 @@ import { CatalogProcessorCache } from '@backstage/plugin-catalog-node';
 import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node';
 import { DiscoveryService } from '@backstage/backend-plugin-api';
 import { Entity } from '@backstage/catalog-model';
+import { EntityProvider } from '@backstage/plugin-catalog-node';
+import { EntityProviderConnection } from '@backstage/plugin-catalog-node';
 import { LocationSpec } from '@backstage/plugin-catalog-common';
+import { MarketplaceCollection } from '@red-hat-developer-hub/backstage-plugin-marketplace-common';
 import { MarketplacePackage } from '@red-hat-developer-hub/backstage-plugin-marketplace-common';
 import { MarketplacePlugin } from '@red-hat-developer-hub/backstage-plugin-marketplace-common';
+import { SchedulerServiceTaskRunner } from '@backstage/backend-plugin-api';
+
+// @public (undocumented)
+export abstract class BaseEntityProvider<T extends Entity> implements EntityProvider {
+    constructor(taskRunner: SchedulerServiceTaskRunner);
+    // (undocumented)
+    connect(connection: EntityProviderConnection): Promise<void>;
+    // (undocumented)
+    getEntities(allEntities: JsonFileData<T>[]): T[];
+    // (undocumented)
+    abstract getKind(): string;
+    // (undocumented)
+    abstract getProviderName(): string;
+    // (undocumented)
+    run(): Promise<void>;
+}
 
 // @public (undocumented)
 export type CachedData = {
@@ -39,6 +58,12 @@ export class DynamicPackageInstallStatusProcessor implements CatalogProcessor {
     preProcessEntity(entity: Entity, _location: LocationSpec, _emit: CatalogProcessorEmit, _originLocation: LocationSpec, cache: CatalogProcessorCache): Promise<Entity>;
 }
 
+// @public (undocumented)
+export type JsonFileData<T> = {
+    filePath: string;
+    content: T;
+};
+
 // @public (undocumented)
 export class LocalPackageInstallStatusProcessor implements CatalogProcessor {
     constructor(paths?: string[]);
@@ -60,6 +85,14 @@ export class MarketplaceCollectionProcessor implements CatalogProcessor {
     validateEntityKind(entity: Entity): Promise<boolean>;
 }
 
+// @public (undocumented)
+export class MarketplaceCollectionProvider extends BaseEntityProvider<MarketplaceCollection> {
+    // (undocumented)
+    getKind(): string;
+    // (undocumented)
+    getProviderName(): string;
+}
+
 // @public (undocumented)
 export class MarketplacePackageProcessor implements CatalogProcessor {
     // (undocumented)
@@ -70,6 +103,14 @@ export class MarketplacePackageProcessor implements CatalogProcessor {
     validateEntityKind(entity: Entity): Promise<boolean>;
 }
 
+// @public (undocumented)
+export class MarketplacePackageProvider extends BaseEntityProvider<MarketplacePackage> {
+    // (undocumented)
+    getKind(): string;
+    // (undocumented)
+    getProviderName(): string;
+}
+
 // @public (undocumented)
 export class MarketplacePluginProcessor implements CatalogProcessor {
     // (undocumented)
@@ -80,4 +121,12 @@ export class MarketplacePluginProcessor implements CatalogProcessor {
     validateEntityKind(entity: Entity): Promise<boolean>;
 }
 
+// @public (undocumented)
+export class MarketplacePluginProvider extends BaseEntityProvider<MarketplacePlugin> {
+    // (undocumented)
+    getKind(): string;
+    // (undocumented)
+    getProviderName(): string;
+}
+
 ```

workspaces/marketplace/plugins/catalog-backend-module-marketplace/src/index.ts

@@ -23,3 +23,5 @@
 export { catalogModuleMarketplace as default } from './module';
 
 export * from './processors';
+export * from './providers';
+export * from './types';

workspaces/marketplace/plugins/catalog-backend-module-marketplace/src/module.test.ts

@@ -20,7 +20,10 @@ import { startTestBackend } from '@backstage/backend-test-utils';
 
 describe('catalogModuleMarketplace', () => {
   it('should register the extension point', async () => {
-    const extensionPoint = { addProcessor: jest.fn() };
+    const extensionPoint = {
+      addProcessor: jest.fn(),
+      addEntityProvider: jest.fn(),
+    };
     await startTestBackend({
       extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]],
       features: [catalogModuleMarketplace],

workspaces/marketplace/plugins/catalog-backend-module-marketplace/src/module.ts

@@ -25,6 +25,8 @@ import { MarketplaceCollectionProcessor } from './processors/MarketplaceCollecti
 import { DynamicPackageInstallStatusProcessor } from './processors/DynamicPackageInstallStatusProcessor';
 import { LocalPackageInstallStatusProcessor } from './processors/LocalPackageInstallStatusProcessor';
 import { MarketplacePackageProcessor } from './processors/MarketplacePackageProcessor';
+import { MarketplacePluginProvider } from './providers/MarketplacePluginProvider';
+import { MarketplacePackageProvider } from './providers/MarketplacePackageProvider';
 
 /**
  * @public
@@ -39,9 +41,23 @@ export const catalogModuleMarketplace = createBackendModule({
         catalog: catalogProcessingExtensionPoint,
         discovery: coreServices.discovery,
         auth: coreServices.auth,
+        scheduler: coreServices.scheduler,
       },
-      async init({ logger, catalog, discovery, auth }) {
-        logger.info('Adding Marketplace processors to catalog...');
+      async init({ logger, catalog, discovery, auth, scheduler }) {
+        logger.info(
+          'Adding Marketplace providers and processors to catalog...',
+        );
+        const taskRunner = scheduler.createScheduledTaskRunner({
+          frequency: { minutes: 30 },
+          timeout: { minutes: 10 },
+        });
+
+        catalog.addEntityProvider(new MarketplacePackageProvider(taskRunner));
+        catalog.addEntityProvider(new MarketplacePluginProvider(taskRunner));
+        // Disabling the collection provider as collections/all.yaml is already commented in RHDH 1.5 image.
+        // catalog.addEntityProvider(
+        //   new MarketplaceCollectionProvider(taskRunner),
+        // );
         catalog.addProcessor(new MarketplacePluginProcessor());
         catalog.addProcessor(new MarketplaceCollectionProcessor());
         catalog.addProcessor(new LocalPackageInstallStatusProcessor());

workspaces/marketplace/plugins/catalog-backend-module-marketplace/src/providers/BaseEntityProvider.ts

@@ -0,0 +1,100 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { SchedulerServiceTaskRunner } from '@backstage/backend-plugin-api';
+import {
+  ANNOTATION_LOCATION,
+  ANNOTATION_ORIGIN_LOCATION,
+  Entity,
+} from '@backstage/catalog-model';
+import {
+  EntityProvider,
+  EntityProviderConnection,
+} from '@backstage/plugin-catalog-node';
+
+import { findTopmostFolder, readYamlFiles } from '../utils/file-utils';
+import { JsonFileData } from '../types';
+
+/**
+ * @public
+ */
+export abstract class BaseEntityProvider<T extends Entity>
+  implements EntityProvider
+{
+  private connection?: EntityProviderConnection;
+  private taskRunner: SchedulerServiceTaskRunner;
+
+  constructor(taskRunner: SchedulerServiceTaskRunner) {
+    this.taskRunner = taskRunner;
+  }
+
+  abstract getProviderName(): string;
+  abstract getKind(): string;
+
+  getEntities(allEntities: JsonFileData<T>[]): T[] {
+    if (allEntities.length === 0) {
+      return [];
+    }
+    return allEntities
+      .filter(d => d.content.kind === this.getKind())
+      .map(file => ({
+        ...file.content,
+        metadata: {
+          ...file.content.metadata,
+          annotations: {
+            [ANNOTATION_LOCATION]: `file:${this.getProviderName()}`,
+            [ANNOTATION_ORIGIN_LOCATION]: `file:${this.getProviderName()}`,
+          },
+        },
+      }));
+  }
+
+  async connect(connection: EntityProviderConnection): Promise<void> {
+    this.connection = connection;
+    await this.taskRunner.run({
+      id: this.getProviderName(),
+      fn: async () => {
+        await this.run();
+      },
+    });
+  }
+
+  async run(): Promise<void> {
+    if (!this.connection) {
+      throw new Error('Not initialized');
+    }
+
+    const marketplaceFilePath = findTopmostFolder('marketplace');
+
+    let yamlData: JsonFileData<T>[] = [];
+    if (marketplaceFilePath) {
+      try {
+        yamlData = readYamlFiles(marketplaceFilePath);
+      } catch (error) {
+        console.error(error.message);
+      }
+    }
+
+    const entities: T[] = this.getEntities(yamlData);
+
+    await this.connection.applyMutation({
+      type: 'full',
+      entities: entities.map(entity => ({
+        entity,
+        locationKey: `file:${this.getProviderName()}`,
+      })),
+    });
+  }
+}

workspaces/marketplace/plugins/catalog-backend-module-marketplace/src/providers/MarketplaceCollectionProvider.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  MarketplaceCollection,
+  MarketplaceKind,
+} from '@red-hat-developer-hub/backstage-plugin-marketplace-common';
+import { BaseEntityProvider } from './BaseEntityProvider';
+
+/**
+ * @public
+ */
+export class MarketplaceCollectionProvider extends BaseEntityProvider<MarketplaceCollection> {
+  getKind(): string {
+    return MarketplaceKind.Collection;
+  }
+
+  getProviderName(): string {
+    return 'marketplace-collection-provider';
+  }
+}

workspaces/marketplace/plugins/catalog-backend-module-marketplace/src/providers/MarketplacePackageProvider.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  MarketplaceKind,
+  MarketplacePackage,
+} from '@red-hat-developer-hub/backstage-plugin-marketplace-common';
+import { BaseEntityProvider } from './BaseEntityProvider';
+
+/**
+ * @public
+ */
+export class MarketplacePackageProvider extends BaseEntityProvider<MarketplacePackage> {
+  getKind(): string {
+    return MarketplaceKind.Package;
+  }
+
+  getProviderName(): string {
+    return 'marketplace-package-provider';
+  }
+}

workspaces/marketplace/plugins/catalog-backend-module-marketplace/src/providers/MarketplacePluginProvider.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  MarketplaceKind,
+  MarketplacePlugin,
+} from '@red-hat-developer-hub/backstage-plugin-marketplace-common';
+import { BaseEntityProvider } from './BaseEntityProvider';
+
+/**
+ * @public
+ */
+export class MarketplacePluginProvider extends BaseEntityProvider<MarketplacePlugin> {
+  getKind(): string {
+    return MarketplaceKind.Plugin;
+  }
+
+  getProviderName(): string {
+    return 'marketplace-plugin-provider';
+  }
+}

workspaces/marketplace/plugins/catalog-backend-module-marketplace/src/providers/index.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export * from './BaseEntityProvider';
+export * from './MarketplacePluginProvider';
+export * from './MarketplaceCollectionProvider';
+export * from './MarketplacePackageProvider';