Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[POC] ProjectBuilder: Build projects concurrently if possible #626

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

[POC] ProjectBuilder: Build projects concurrently if possible #626

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
04ce22c
[POC] ProjectBuilder: Build projects in parallel if possible
RandomByte Jun 22, 2023
b2e9c67
Optimize determination for parallel processing slightly
RandomByte Jun 22, 2023
bcca6ff
Add UI5_BUILD_PARALLEL env var to enable new behavior
RandomByte Jun 22, 2023
1f5293f
Further optimize for maximum concurrency
RandomByte Jun 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 88 additions & 63 deletions lib/build/ProjectBuilder.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ class ProjectBuilder {
* @param {object} parameters Parameters
* @param {string} parameters.destPath Target path
* @param {boolean} [parameters.cleanDest=false] Decides whether project should clean the target path before build
* @param {boolean} [parameters.concurrentBuild=false] Whether to build projects concurrently if possible
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

todo: we should document well what it means "if possible"

* @param {Array.<string|RegExp>} [parameters.includedDependencies=[]]
* List of names of projects to include in the build result
* If the wildcard '*' is provided, all dependencies will be included in the build result.
Expand All @@ -133,7 +134,7 @@ class ProjectBuilder {
* @returns {Promise} Promise resolving once the build has finished
*/
async build({
destPath, cleanDest = false, parallel = !!process.env.UI5_BUILD_PARALLEL,
destPath, cleanDest = false, concurrentBuild = !!process.env.UI5_BUILD_CONCURRENT,
includedDependencies = [], excludedDependencies = [],
dependencyIncludes
}) {
Expand Down Expand Up @@ -170,61 +171,46 @@ class ProjectBuilder {
`while including any dependencies into the build result`);
}

const buildContexts = await this._createRequiredBuildContexts(requestedProjects);
const projectContexts = await this._createRequiredBuildContexts(requestedProjects);
const cleanupSigHooks = this._registerCleanupSigHooks();
const fsTarget = resourceFactory.createAdapter({
fsBasePath: destPath,
virBasePath: "/"
});

const queue = []; // A queue of batches to process
const alreadyBuilt = [];

function determineBatchNumber(depName, minBatchNumber) {
for (let i = queue.length - 1; i >= minBatchNumber; i--) {
if (queue[i].has(depName)) {
return i + 1;
const pendingDependencies = new Map();
const pendingProjects = new Set();

// Start with a deep graph traversal to get a deterministic (and somewhat reasonable) build order
await this._graph.traverseDepthFirst(({project}) => {
const projectName = project.getName();
const {projectBuildContext, requiredDependencies} = projectContexts.get(projectName);
if (projectBuildContext) {
// Build context exists
// => This project needs to be built or, in case it has already
// been built, it's build result needs to be written out (if requested)
pendingProjects.add(projectName);
pendingDependencies.set(projectName, new Set(requiredDependencies));

if (!projectBuildContext.requiresBuild()) {
alreadyBuilt.push(projectName);
}
}
return minBatchNumber;
}

buildContexts.forEach(({projectBuildContext, requiredDependencies}, projectName) => {
// => This project needs to be built or, in case it has already
// been built, it's build result needs to be written out (if requested)
if (!projectBuildContext.requiresBuild()) {
alreadyBuilt.push(projectName);
}
let minBatchNumber = queue.length;
if (parallel) {
minBatchNumber = 0;
if (queue.length) {
for (const depName of requiredDependencies) {
minBatchNumber = determineBatchNumber(depName, minBatchNumber);
}
}
}

console.log(`${projectName}: Queued at batch #${minBatchNumber}`);
if (minBatchNumber === queue.length) {
// Add new batch to queue
queue.push(new Map());
}
queue[minBatchNumber].set(projectName, projectBuildContext);
});

log.setProjects(Array.from(buildContexts.keys()));
if (queue.length > 1) { // Do not log if only the root project is being built
log.info(`Processing ${buildContexts.size} projects`);
log.setProjects(Array.from(pendingProjects.keys()));
if (pendingProjects.size > 1) { // Do not log if only the root project is being built
log.info(`Processing ${pendingProjects.size} projects`);
if (alreadyBuilt.length) {
log.info(` Reusing build results of ${alreadyBuilt.length} projects`);
log.info(` Building ${buildContexts.size - alreadyBuilt.length} projects`);
log.info(` Building ${pendingProjects.size - alreadyBuilt.length} projects`);
}

if (log.isLevelEnabled("verbose")) {
log.verbose(` Required projects:`);
log.verbose(` ${queue
.map((projectBuildContext) => {
log.verbose(` Required projects:\n ${Array.from(projectContexts.values())
.map(({projectBuildContext}) => {
const projectName = projectBuildContext.getProject().getName();
let msg;
if (alreadyBuilt.includes(projectName)) {
Expand All @@ -240,39 +226,78 @@ class ProjectBuilder {
}
}

const projectsInProcess = new Set();
function hasPendingDependencies(projectName) {
const pendingDeps = pendingDependencies.get(projectName);
for (const depName of pendingDeps) {
if (!pendingProjects.has(depName) && !projectsInProcess.has(depName)) {
pendingDeps.delete(depName);
}
}

if (log.isLevelEnabled("verbose")) {
log.verbose(`${projectName} is waiting for: ${Array.from(pendingDeps.keys()).join(", ")}`);
}
return !!pendingDeps.size;
}

if (cleanDest) {
log.info(`Cleaning target directory...`);
await rimraf(destPath);
}
const startTime = process.hrtime();
try {
const pBuilds = new Map();
const pWrites = [];
for (const batch of queue) {
const projectBuildContexts = Array.from(batch.values());
log.info(`Building ${projectBuildContexts.length} projects in parallel:`);
await Promise.all(projectBuildContexts.map(async (projectBuildContext) => {
const projectName = projectBuildContext.getProject().getName();
const projectType = projectBuildContext.getProject().getType();
log.verbose(`Processing project ${projectName}...`);

// Only build projects that are not already build (i.e. provide a matching build manifest)
if (alreadyBuilt.includes(projectName)) {
log.skipProjectBuild(projectName, projectType);
} else {
log.startProjectBuild(projectName, projectType);
await projectBuildContext.getTaskRunner().runTasks();
log.endProjectBuild(projectName, projectType);
}
if (!requestedProjects.includes(projectName)) {
// Project has not been requested
// => Its resources shall not be part of the build result
return;
}

log.verbose(`Writing out files...`);
pWrites.push(this._writeResults(projectBuildContext, fsTarget));
}));
while (pendingProjects.size) {
const buildCount = pBuilds.length;
for (const projectName of pendingProjects) {
if (!hasPendingDependencies(projectName)) {
projectsInProcess.add(projectName);
pendingProjects.delete(projectName);
// => Build the project
pBuilds.set(projectName, (async ({projectBuildContext}) => {
const projectName = projectBuildContext.getProject().getName();
const projectType = projectBuildContext.getProject().getType();

// Only build projects that are not already build (i.e. provide a matching build manifest)
if (alreadyBuilt.includes(projectName)) {
log.skipProjectBuild(projectName, projectType);
} else {
log.startProjectBuild(projectName, projectType);
await projectBuildContext.getTaskRunner().runTasks();
log.endProjectBuild(projectName, projectType);
}
projectsInProcess.delete(projectName);
if (!requestedProjects.includes(projectName)) {
// Project has not been requested
// => Its resources shall not be part of the build result
return;
}

log.verbose(`Writing out files...`);
pWrites.push(this._writeResults(projectBuildContext, fsTarget));
return projectName;
})(projectContexts.get(projectName)));

if (!concurrentBuild) {
// Do not add more builds
break;
}
}
}
if (buildCount === pBuilds.length || !concurrentBuild) {
log.verbose(`Waiting for a build to finish`);
// Either schedule a build or wait till any promise resolved
const finishedProjectName = await Promise.any(pBuilds.values());
pBuilds.delete(finishedProjectName);
log.verbose(`${finishedProjectName} finished building. Builds in process: ${pBuilds.size}`);
}
}
// Wait for any remaining builds
await Promise.all(pBuilds);
// Wait for the deferred writes
await Promise.all(pWrites);
log.info(`Build succeeded in ${this._getElapsedTime(startTime)}`);
} catch (err) {
Expand Down