Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[POC] ProjectBuilder: Build projects concurrently if possible #626

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

[POC] ProjectBuilder: Build projects concurrently if possible #626

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
04ce22c
[POC] ProjectBuilder: Build projects in parallel if possible
RandomByte Jun 22, 2023
b2e9c67
Optimize determination for parallel processing slightly
RandomByte Jun 22, 2023
bcca6ff
Add UI5_BUILD_PARALLEL env var to enable new behavior
RandomByte Jun 22, 2023
1f5293f
Further optimize for maximum concurrency
RandomByte Jun 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 89 additions & 39 deletions lib/build/ProjectBuilder.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ class ProjectBuilder {
* @param {object} parameters Parameters
* @param {string} parameters.destPath Target path
* @param {boolean} [parameters.cleanDest=false] Decides whether project should clean the target path before build
* @param {boolean} [parameters.concurrentBuild=false] Whether to build projects concurrently if possible
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

todo: we should document well what it means "if possible"

* @param {Array.<string|RegExp>} [parameters.includedDependencies=[]]
* List of names of projects to include in the build result
* If the wildcard '*' is provided, all dependencies will be included in the build result.
Expand All @@ -133,7 +134,7 @@ class ProjectBuilder {
* @returns {Promise} Promise resolving once the build has finished
*/
async build({
destPath, cleanDest = false,
destPath, cleanDest = false, concurrentBuild = !!process.env.UI5_BUILD_CONCURRENT,
includedDependencies = [], excludedDependencies = [],
dependencyIncludes
}) {
Expand Down Expand Up @@ -170,45 +171,46 @@ class ProjectBuilder {
`while including any dependencies into the build result`);
}

const projectBuildContexts = await this._createRequiredBuildContexts(requestedProjects);
const projectContexts = await this._createRequiredBuildContexts(requestedProjects);
const cleanupSigHooks = this._registerCleanupSigHooks();
const fsTarget = resourceFactory.createAdapter({
fsBasePath: destPath,
virBasePath: "/"
});

const queue = [];
const alreadyBuilt = [];

// Create build queue based on graph depth-first search to ensure correct build order
await this._graph.traverseDepthFirst(async ({project}) => {
const pendingDependencies = new Map();
const pendingProjects = new Set();

// Start with a deep graph traversal to get a deterministic (and somewhat reasonable) build order
await this._graph.traverseDepthFirst(({project}) => {
const projectName = project.getName();
const projectBuildContext = projectBuildContexts.get(projectName);
const {projectBuildContext, requiredDependencies} = projectContexts.get(projectName);
if (projectBuildContext) {
// Build context exists
// => This project needs to be built or, in case it has already
// been built, it's build result needs to be written out (if requested)
queue.push(projectBuildContext);
pendingProjects.add(projectName);
pendingDependencies.set(projectName, new Set(requiredDependencies));

if (!projectBuildContext.requiresBuild()) {
alreadyBuilt.push(projectName);
}
}
});

log.setProjects(queue.map((projectBuildContext) => {
return projectBuildContext.getProject().getName();
}));
if (queue.length > 1) { // Do not log if only the root project is being built
log.info(`Processing ${queue.length} projects`);
log.setProjects(Array.from(pendingProjects.keys()));
if (pendingProjects.size > 1) { // Do not log if only the root project is being built
log.info(`Processing ${pendingProjects.size} projects`);
if (alreadyBuilt.length) {
log.info(` Reusing build results of ${alreadyBuilt.length} projects`);
log.info(` Building ${queue.length - alreadyBuilt.length} projects`);
log.info(` Building ${pendingProjects.size - alreadyBuilt.length} projects`);
}

if (log.isLevelEnabled("verbose")) {
log.verbose(` Required projects:`);
log.verbose(` ${queue
.map((projectBuildContext) => {
log.verbose(` Required projects:\n ${Array.from(projectContexts.values())
.map(({projectBuildContext}) => {
const projectName = projectBuildContext.getProject().getName();
let msg;
if (alreadyBuilt.includes(projectName)) {
Expand All @@ -224,35 +226,78 @@ class ProjectBuilder {
}
}

const projectsInProcess = new Set();
function hasPendingDependencies(projectName) {
const pendingDeps = pendingDependencies.get(projectName);
for (const depName of pendingDeps) {
if (!pendingProjects.has(depName) && !projectsInProcess.has(depName)) {
pendingDeps.delete(depName);
}
}

if (log.isLevelEnabled("verbose")) {
log.verbose(`${projectName} is waiting for: ${Array.from(pendingDeps.keys()).join(", ")}`);
}
return !!pendingDeps.size;
}

if (cleanDest) {
log.info(`Cleaning target directory...`);
await rimraf(destPath);
}
const startTime = process.hrtime();
try {
const pBuilds = new Map();
const pWrites = [];
for (const projectBuildContext of queue) {
const projectName = projectBuildContext.getProject().getName();
const projectType = projectBuildContext.getProject().getType();
log.verbose(`Processing project ${projectName}...`);

// Only build projects that are not already build (i.e. provide a matching build manifest)
if (alreadyBuilt.includes(projectName)) {
log.skipProjectBuild(projectName, projectType);
} else {
log.startProjectBuild(projectName, projectType);
await projectBuildContext.getTaskRunner().runTasks();
log.endProjectBuild(projectName, projectType);

while (pendingProjects.size) {
const buildCount = pBuilds.length;
for (const projectName of pendingProjects) {
if (!hasPendingDependencies(projectName)) {
projectsInProcess.add(projectName);
pendingProjects.delete(projectName);
// => Build the project
pBuilds.set(projectName, (async ({projectBuildContext}) => {
const projectName = projectBuildContext.getProject().getName();
const projectType = projectBuildContext.getProject().getType();

// Only build projects that are not already build (i.e. provide a matching build manifest)
if (alreadyBuilt.includes(projectName)) {
log.skipProjectBuild(projectName, projectType);
} else {
log.startProjectBuild(projectName, projectType);
await projectBuildContext.getTaskRunner().runTasks();
log.endProjectBuild(projectName, projectType);
}
projectsInProcess.delete(projectName);
if (!requestedProjects.includes(projectName)) {
// Project has not been requested
// => Its resources shall not be part of the build result
return;
}

log.verbose(`Writing out files...`);
pWrites.push(this._writeResults(projectBuildContext, fsTarget));
return projectName;
})(projectContexts.get(projectName)));

if (!concurrentBuild) {
// Do not add more builds
break;
}
}
}
if (!requestedProjects.includes(projectName)) {
// Project has not been requested
// => Its resources shall not be part of the build result
continue;
if (buildCount === pBuilds.length || !concurrentBuild) {
log.verbose(`Waiting for a build to finish`);
// Either schedule a build or wait till any promise resolved
const finishedProjectName = await Promise.any(pBuilds.values());
pBuilds.delete(finishedProjectName);
log.verbose(`${finishedProjectName} finished building. Builds in process: ${pBuilds.size}`);
}

log.verbose(`Writing out files...`);
pWrites.push(this._writeResults(projectBuildContext, fsTarget));
}
// Wait for any remaining builds
await Promise.all(pBuilds);
// Wait for the deferred writes
await Promise.all(pWrites);
log.info(`Build succeeded in ${this._getElapsedTime(startTime)}`);
} catch (err) {
Expand All @@ -269,26 +314,31 @@ class ProjectBuilder {
return requestedProjects.includes(projectName);
}));

const projectBuildContexts = new Map();
const buildContexts = new Map();

for (const projectName of requiredProjects) {
log.verbose(`Creating build context for project ${projectName}...`);
const projectBuildContext = this._buildContext.createProjectContext({
project: this._graph.getProject(projectName)
});

projectBuildContexts.set(projectName, projectBuildContext);
const projectInfo = {
projectBuildContext,
requiredDependencies: new Set()
};
buildContexts.set(projectName, projectInfo);

if (projectBuildContext.requiresBuild()) {
const taskRunner = projectBuildContext.getTaskRunner();
const requiredDependencies = await taskRunner.getRequiredDependencies();
projectInfo.requiredDependencies = requiredDependencies;

if (requiredDependencies.size === 0) {
continue;
}
// This project needs to be built and required dependencies to be built as well
this._graph.getDependencies(projectName).forEach((depName) => {
if (projectBuildContexts.has(depName)) {
if (buildContexts.has(depName)) {
// Build context already exists
// => Dependency will be built
return;
Expand All @@ -302,7 +352,7 @@ class ProjectBuilder {
}
}

return projectBuildContexts;
return buildContexts;
}

async _getProjectFilter({
Expand Down