Skip to content

Commit

Permalink
ZFIN-9476: Improve the speed of gradle make (#1260)
Browse files Browse the repository at this point in the history
* Remove rss files.
We don't have any traffic to our rss feeds and even if we did, they redirect to a 404 page at atlassian.
These are the only files in zf_info that use "<!--|" type placeholders.

* Remove rss generating r scripts

* Introduce new syntax and class (SimpleDirectoryCopyTask.groovy) for copying files (server_apps, cgi-bin, and home) when invoking `gradle make`

* Removed tasks for subdirectories of server_apps/data_transfer that are already covered by the parent directory task. Ensembl is the only one that remains, though that too might be unnecessary
  • Loading branch information
rtaylorzfin authored Dec 13, 2024
1 parent bb6e868 commit 3cee642
Show file tree
Hide file tree
Showing 10 changed files with 279 additions and 792 deletions.
531 changes: 153 additions & 378 deletions build.gradle

Large diffs are not rendered by default.

126 changes: 126 additions & 0 deletions buildSrc/src/main/groovy/SimpleDirectoryCopyTask.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.Internal
import org.gradle.api.tasks.TaskAction
import org.gradle.api.GradleException

class SimpleDirectoryCopyTask extends DefaultTask {

@Input
String sourcePath

@Input
List<String> includes = [] // List of file extensions to include (e.g., 'sh', 'sql')

@Input
List<String> excludes = [] // List of filenames to exclude (e.g., 'Makefile', 'build.gradle')

@Input
List<String> excludeDirs = [] // List of directories to exclude (relative to sourcePath)

// Access global properties from the project.ext
@Internal
def targetroot = project.ext.targetroot

@Internal
def ttNameMap = project.ext.ttNameMap

@Internal
def whitelistTemplateFiles = project.ext.whitelistTemplateFiles

@TaskAction
void copyFiles() {
def debugMode = false
println "Starting copy task for sourcePath: $sourcePath"

def targetDir = new File("$targetroot/$sourcePath")
def sourceDir = new File(sourcePath)

if (!sourceDir.exists()) {
println "Source directory does not exist: $sourceDir"
return
}

println "Exclude dirs: " + (excludeDirs.isEmpty() ? "None" : excludeDirs)
println "Include extensions: " + (includes.isEmpty() ? "All" : includes)
println "Exclude extensions: " + (excludes.isEmpty() ? "None" : excludes)
println "--------------------------------------------------"

def copySingleFile = { source, destination ->
def relativeDestination = sourceDir.toURI().relativize(source.toURI()).path

//are the files the same?
if (destination.exists()) {
def diffProc = ["diff", source, destination].execute()
diffProc.waitForOrKill(3600)
if (diffProc.exitValue() == 0) {
if (debugMode) {
println " Skipped: $relativeDestination (Identical file)"
}
return
}
}

//if not, copy the file
def sout = new StringBuilder(), serr = new StringBuilder()
def proc = ["cp", source, destination].execute()
proc.consumeProcessOutput(sout, serr)
proc.waitForOrKill(3600)
if (proc.exitValue() != 0) {
println "Error"
println "$serr"
throw new GradleException("Failed to copy file: $source")
}
println " Copied: $relativeDestination"
// println "out> $sout\nerr> $serr"
}

sourceDir.eachFileRecurse { file ->
def relativePath = sourceDir.toURI().relativize(file.toURI()).path
def pathRelativeToThisBuildFile = sourcePath + '/' + relativePath

// Skip files in excluded directories (or if any of the excluded directories is a parent/grandparent/etc. directory)
if (excludeDirs.any { excludedDir -> relativePath.startsWith("${excludedDir}/") || relativePath == excludedDir }) {
if (debugMode) {
println " Skipped: $relativePath (Excluded directory or subdirectory)"
}
return
}

// Filter files based on includes and excludes
def extension = file.name.tokenize('.').last()
def filename = file.name
if ((includes.isEmpty() || includes.contains(extension)) && !excludes.contains(filename)) {
def destinationFile = new File(targetDir, relativePath)
destinationFile.parentFile.mkdirs()

if (file.isDirectory()) {
if (!destinationFile.exists()) {
destinationFile.mkdirs()
println " Directory: $relativePath"
}
} else if (whitelistTemplateFiles.contains(pathRelativeToThisBuildFile)) {
// Perform search-and-replace for whitelisted files
def processedContent = file.text
ttNameMap.each { name, value ->
processedContent = processedContent.replaceAll("<!--\\|${name}\\|-->", value)
}
if (!destinationFile.exists() || destinationFile.text != processedContent) {
destinationFile.text = processedContent
println "Transformed: $relativePath"
} else {
if (debugMode) {
println " Skipped: $relativePath (Identical file after transform)"
}
}
} else {
copySingleFile(file, destinationFile)
}
} else {
if (debugMode) {
println " Skipped: $relativePath (Skipped by includes/excludes)"
}
}
}
}
}
5 changes: 0 additions & 5 deletions home/zf_info/news/jobs.rss

This file was deleted.

5 changes: 0 additions & 5 deletions home/zf_info/news/mtgs.rss

This file was deleted.

4 changes: 0 additions & 4 deletions home/zf_info/news/siteNews.rss

This file was deleted.

52 changes: 0 additions & 52 deletions server_apps/WebSiteTools/gen-jobs-rss.r

This file was deleted.

59 changes: 0 additions & 59 deletions server_apps/WebSiteTools/gen-meetings-rss.r

This file was deleted.

69 changes: 0 additions & 69 deletions server_apps/WebSiteTools/gen-news-rss.r

This file was deleted.

Loading

0 comments on commit 3cee642

Please sign in to comment.