diff --git a/devel/analysis/analysis.py b/devel/analysis/analysis.py index 83088212a..ab85c171d 100644 --- a/devel/analysis/analysis.py +++ b/devel/analysis/analysis.py @@ -2,6 +2,7 @@ from raytkUtil import RaytkContext, ROPInfo, InputInfo, CategoryInfo import raytkDocs +from raytkState import RopState # noinspection PyUnreachableCode if False: @@ -136,7 +137,7 @@ def buildOpParamsTable(dat: DAT): ]) for tuplet in info.rop.customTuplets: par = tuplet[0] - if par.name in ('Inspect', 'Help', 'Updateop') or par.name.startswith('Createref') or par.name.startswith('Creatersel'): + if par.name in ('Inspect', 'Help', 'Updateop') or par.name.startswith('Createref'): continue cell = dat[info.path, par.tupletName] if cell is None: @@ -223,10 +224,18 @@ def buildOpCurrentExpandedParamsTable(dat: DAT): info = ROPInfo(rop) if not info or not info.isROP: continue - expanded = ' '.join([ - cell.val - for cell in info.opDef.op('paramSpecTable').col('localName')[1:] - ]) + opDefExt = info.opDefExt + if not opDefExt: + print('Error loading state for', rop) + continue + state = opDefExt.getRopState() + if hasattr(state, 'params') and state.params: + expanded = [ + p.localName + for p in state.params + ] + else: + expanded = [] dat.appendRow([ info.path, expanded, @@ -294,7 +303,7 @@ def buildOpTagTable(dat: DAT): opTagExprs = {} # type: dict[str, dict[str, str]] for rop in RaytkContext().allMasterOperators(): info = ROPInfo(rop) - if not info or not info.isROP: + if not info or not info.isROP or not info.opDefPar['Tagtable']: continue tagTable = info.opDefPar.Tagtable.eval() if not tagTable: diff --git a/devel/analysis/analysis.tox b/devel/analysis/analysis.tox index 909d01c19..84ca96618 100644 Binary files a/devel/analysis/analysis.tox and b/devel/analysis/analysis.tox differ diff --git a/devel/build/build.tox b/devel/build/build.tox index 1cdfc6b94..c279452d2 100644 Binary files a/devel/build/build.tox and b/devel/build/build.tox differ diff --git a/devel/build/buildAsyncExt.py b/devel/build/buildAsyncExt.py new file mode 100644 index 000000000..f33ab8861 --- /dev/null +++ b/devel/build/buildAsyncExt.py @@ -0,0 +1,568 @@ +import zipfile + +import shutil + +from datetime import datetime +from pathlib import Path +from raytkBuild import BuildContext, DocProcessor, chunked_iterable +from raytkTools import RaytkTools +from raytkUtil import RaytkContext, CategoryInfo, IconColors, RaytkTags, focusFirstCustomParameterPage, navigateTo +from typing import Optional, TextIO + +# noinspection PyUnreachableCode +if False: + # noinspection PyUnresolvedReferences + from _stubs import * + from devel.thirdParty.TDAsyncIO import TDAsyncIO + op.TDAsyncIO = TDAsyncIO(COMP()) + +class BuildManagerAsync: + def __init__(self, ownerComp: COMP): + self.ownerComp = ownerComp + self.logTable = ownerComp.op('log') + self.context = None # type: BuildContext | None + self.docProcessor = None # type: DocProcessor | None + self.experimentalMode = False + self.logFile = None # type: TextIO | None + self.enableVerboseLogging = False + + def OnInit(self): + self.ClearLog() + + @staticmethod + def GetToolkitVersion(): + return RaytkContext().toolkitVersion() + + @staticmethod + def OpenToolkitNetwork(): + navigateTo(RaytkContext().toolkit(), name='raytkBuildNetwork', popup=True, goInto=True) + + def OpenLog(self): + dat = self.ownerComp.op('full_log_text') + dat.openViewer() + + def ClearLog(self): + self.logTable.clear() + self.closeLogFile() + + def closeLogFile(self): + if not self.logFile: + return + self.logFile.close() + self.logFile = None + + def startNewLogFile(self): + stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') + fileName = f'build/log/build-{stamp}.txt' + filePath = Path(fileName) + filePath.parent.mkdir(parents=True, exist_ok=True) + self.logFile = filePath.open('a') + + def ReloadToolkit(self): + self.logTable.clear() + self.log('Reloading toolkit') + builder = ToolkitBuilderAsync(self.context) + op.TDAsyncIO.Run([builder.reloadToolkit()]) + + def prepareForBuild(self): + self.experimentalMode = bool(self.ownerComp.op('experimental_toggle').par.Value0) + self.enableVerboseLogging = bool(self.ownerComp.op('verboseLogging_toggle').par.Value0) + self.logTable.clear() + self.closeLogFile() + if self.ownerComp.op('useLogFile_toggle').par.Value0: + self.startNewLogFile() + self.context = BuildContext(self.log, self.experimentalMode) + + def runToolkitBuildOnly(self): + self.prepareForBuild() + builder = ToolkitBuilderAsync(self.context) + + async def _build(): + await builder.runBuild() + self.log('Finished toolkit build process') + self.closeLogFile() + + op.TDAsyncIO.Cancel() + op.TDAsyncIO.Run([_build()]) + + def runSnippetBuildOnly(self): + self.prepareForBuild() + builder = SnippetsBuilderAsync(self.context) + + async def _build(): + await builder.runBuild() + self.log('Finished snippet build process') + self.closeLogFile() + + op.TDAsyncIO.Cancel() + op.TDAsyncIO.Run([_build()]) + + def runToolkitAndSnippetBuilds(self): + self.prepareForBuild() + + async def _build(): + toolkitBuilder = ToolkitBuilderAsync(self.context) + await toolkitBuilder.runBuild() + self.log('Finished toolkit build process') + + snippetBuilder = SnippetsBuilderAsync(self.context) + await snippetBuilder.runBuild() + self.log('Finished snippet build process') + + op.TDAsyncIO.Cancel() + op.TDAsyncIO.Run([_build()]) + + def log(self, message: str, verbose=False): + if verbose and not self.enableVerboseLogging: + return + print(message, flush=True) + if self.logFile: + stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') + print(stamp, message, file=self.logFile, flush=True) + self.logTable.appendRow([message]) + +class BuilderAsyncBase: + def __init__(self, context: BuildContext): + self.context = context + + def _getOutputToxPath(self, baseName: str): + return self._getOutputFilePath(baseName, 'tox') + + def _getOutputFilePath(self, baseName: str, extension: str): + version = RaytkContext().toolkitVersion() + if self.context.experimental: + suffix = '-exp' + else: + suffix = '' + return f'build/{baseName}-{version}{suffix}.{extension}' + + def log(self, message: str, verbose=False): + self.context.log(message, verbose) + + async def logStageStart(self, desc: str): + self.log(f'===[{desc}]===') + await _asyncYield() + +class ToolkitBuilderAsync(BuilderAsyncBase): + def __init__(self, context: BuildContext): + super().__init__(context) + self.docProcessor = None + if not self.context.experimental: + self.docProcessor = DocProcessor( + context, + dataFolder='docs/_data', + referenceFolder='docs/_reference', + imagesFolder='docs/assets/images', + ) + self.toolkit = None # type: COMP | None + + async def runBuild(self): + self.log('Starting build') + await self.reloadToolkit() + version = RaytkContext().toolkitVersion() + self.log(f'Version: {version}' + (' (experimental)' if self.context.experimental else '')) + self.context.openNetworkPane() + + await self._detachFileSyncOps() + + if self.docProcessor: + await self.logStageStart('Clearing old docs') + self.docProcessor.clearPreviousDocs() + + await self.logStageStart('Process thumbnails') + await self.context.runBuildScript(self.toolkit.op('libraryThumbs/BUILD')) + + await self.logStageStart('Update library info') + await self._updateLibraryInfo() + + await self.logStageStart('Update library image') + await self._updateLibraryImage() + + await self.logStageStart('Pre-process components') + await self._preProcessComponents() + + await self.logStageStart('Process operators') + await self._processAllOperators() + + await self.logStageStart('Process nested operators') + await self._processAllNestedOperators() + + await self.logStageStart('Generate operator docs') + await self._generateAllOperatorDocs() + + await self.logStageStart('Generate category docs') + await self._generateAllCategoryDocs() + + await self.logStageStart('Lock library info') + await self._lockLibraryInfo() + + await self.logStageStart('Process tools') + await self._processTools() + + await self.logStageStart('Lock build lock ops') + await self._lockAllBuildLockOps() + + await self.logStageStart('Destroy components') + await self._destroyComponents() + + await self.logStageStart('Clean operators') + await self._cleanAllOperators() + + await self.logStageStart('Clear operator categories') + await self._cleanAllCategories() + + await self.logStageStart('Consolidate shared python mods') + await self._consolidateSharedPythonMods() + + await self.logStageStart('Remove build exclude ops') + await self._removeAllBuildExcludeOps(self.toolkit) + + await self.logStageStart('Finalize toolkit pars') + await self._finalizeToolkitPars() + + await self.logStageStart('Generate toolkit docs') + await self._generateToolkitDocs() + + await self.logStageStart('Finish build') + await self._finishBuild() + + async def reloadToolkit(self): + self.toolkit = RaytkContext().toolkit() + self.toolkit.par.externaltox = 'src/raytk.tox' + self.toolkit.par.reinitnet.pulse() + # Do this early since it switches off things like automatically writing to the opList.txt file. + # See https://github.com/t3kt/raytk/issues/95 + self.toolkit.par.Devel = False + + async def _detachFileSyncOps(self): + self.context.detachAllFileSyncDatsIn(self.toolkit, reloadFirst=True) + + async def _updateLibraryInfo(self): + if self.toolkit.par['Experimentalbuild'] is not None: + self.toolkit.par.Experimentalbuild.val = self.context.experimental + self.toolkit.par.Experimentalbuild.readOnly = True + libraryInfo = self.toolkit.op('libraryInfo') + libraryInfo.par.Forcebuild.pulse() + self.context.moveNetworkPane(libraryInfo) + await self.context.runBuildScript(libraryInfo.op('BUILD')) + + async def _updateLibraryImage(self): + image = RaytkContext().libraryImage() + if image: + self.context.detachTox(image) + self.context.lockBuildLockOps(image) + image.par.Showshortcut = True + self.toolkit.par.opviewer.val = image + self.toolkit.par.opviewer.readOnly = True + self.toolkit.viewer = True + else: + self.toolkit.par.opviewer.val = '' + self.toolkit.par.opviewer.readOnly = False + self.toolkit.viewer = False + + async def _preProcessComponents(self): + components = self.toolkit.op('components') + shared = components.op('shared') + self.context.lockBuildLockOps(shared) + # this one has a buildLock typeTable that isn't instance-specific, + # so we lock it first then delete the stuff that built it + typeSpec = components.op('typeSpec') + self.context.lockBuildLockOps(typeSpec) + opDef = components.op('opDefinition') + self.context.lockBuildLockOps(opDef) + opDef.op('buildOpState').par.Pretty = False + comps = components.ops( + 'typeSpec', 'typeResolver', 'typeRestrictor', + # 'opImage', # this has some instance-dependent stuff + 'opDefinition', 'compDefinition', + # 'inputHandler', # don't process inputHandler since it uses buildExclude for the config tables, which + # won't have been locked yet when this stage runs + 'shaderBuilder', + 'opElement', 'transformCodeGenerator', 'timeProvider', + 'supportDetector', 'expresssionSwitcher', 'parMenuUpdater', + # 'codeSwitcher', + 'aggregateCodeGenerator', + # 'combiner', # don't process combiner since it has instance-specific buildLock things depending + # on buildExclude + 'waveFunction', + ) + for comp in comps: + self.context.removeBuildExcludeOps(comp) + await _asyncYield() + + async def _processAllOperators(self): + operatorsComp = self.toolkit.op('operators') + self.context.moveNetworkPane(operatorsComp) + self.context.detachTox(operatorsComp) + categories = RaytkContext().allCategories() + if self.docProcessor: + self.docProcessor.writeCategoryListPage(categories) + for category in categories: + await self._processOperatorCategory(category) + + async def _processOperatorCategory(self, category: COMP): + self.log('Processing category: ' + category.path) + categoryInfo = CategoryInfo(category) + self.context.moveNetworkPane(category) + self.context.detachTox(category) + template = category.op('__template') + self.context.safeDestroyOp(template) + if not self.context.experimental: + self.context.removeAlphaOps(category) + for comp in categoryInfo.operators: + await self._processOperator(comp) + + async def _processOperator(self, comp: COMP): + self.log('Processing operator: ' + comp.path) + self.context.focusInNetworkPane(comp) + self.context.disableCloning(comp) + self.context.detachTox(comp) + tools = RaytkTools() + tools.updateROPMetadata(comp) + tools.updateROPParams(comp) + self.context.applyParamUpdatersIn(comp) + self.context.resetCustomPars(comp) + self.context.lockROPPars(comp) + await self._processOperatorSubCompChildren(comp) + if not comp.isPanel: + comp.showCustomOnly = True + self.log('Updating OP image for ' + comp.path) + img = tools.updateOPImage(comp) + if img: + self.context.disableCloning(img) + self.context.detachTox(img) + self.context.lockBuildLockOps(img) + self.context.cleanOpImage(img) + comp.color = IconColors.defaultBgColor + + async def _processOperatorSubCompChildren(self, comp: COMP): + subComps = comp.findChildren(type=COMP) + if not subComps: + return + self.log(f'Processing {len(subComps)} sub-comps of {comp.path}', verbose=True) + for child in subComps: + self.log('Processing sub-comp: ' + child.path, verbose=True) + self.context.detachTox(child) + self.context.reclone(child) + self.context.disableCloning(child) + await self._processOperatorSubCompChildren(child) + + async def _processAllNestedOperators(self): + operatorsComp = self.toolkit.op('operators') + subOps = operatorsComp.findChildren(tags=[RaytkTags.raytkOP.name], depth=3) + self.log(f'Found {len(subOps)} nested operators') + for subOp in subOps: + self.log('Processing sub-operator: ' + subOp.path) + self.context.updateOrReclone(subOp) + self.context.detachTox(subOp) + self.context.disableCloning(subOp) + + async def _generateAllOperatorDocs(self): + if not self.docProcessor: + return + self.log('Generate operator docs') + for comp in RaytkContext().allMasterOperators(): + self.docProcessor.processOp(comp) + + async def _generateAllCategoryDocs(self): + if not self.docProcessor: + return + self.log('Generate category docs') + for comp in RaytkContext().allCategories(): + self.docProcessor.processOpCategory(comp) + + async def _lockLibraryInfo(self): + self.context.lockOps(self.toolkit.ops( + 'info', 'opTable', 'opCategoryTable', 'opHelpTable', 'buildInfo')) + + async def _processTools(self): + tools = self.toolkit.op('tools') + self.context.moveNetworkPane(tools) + self.context.reloadTox(tools) + self.context.detachTox(tools) + await self.context.runBuildScript(tools.op('BUILD')) + + async def _lockAllBuildLockOps(self): + self.context.lockBuildLockOps(self.toolkit) + + async def _destroyComponents(self): + comp = self.toolkit.op('components') + self.context.focusInNetworkPane(comp) + self.context.safeDestroyOp(comp) + + async def _cleanAllOperators(self): + for comp in RaytkContext().allMasterOperators(): + self.log('Clean operator ' + comp.path) + self.context.removeOpHelp(comp) + self.context.cleanOperatorTypeSpecs(comp) + self.context.cleanOperatorDefPars(comp) + + async def _cleanAllCategories(self): + for comp in RaytkContext().allCategories(): + self.log('Clean category ' + comp.path) + self.context.removeCatHelp(comp) + + async def _consolidateSharedPythonMods(self): + self.context.removeRedundantPythonModules(self.toolkit, self.toolkit.ops('tools', 'libraryInfo')) + + async def _removeAllBuildExcludeOps(self, scope: COMP): + self.log(f'Removing buildExclude ops in {scope}') + toRemove = scope.findChildren(tags=[RaytkTags.buildExclude.name], includeUtility=True) + chunks = [list(chunk) for chunk in chunked_iterable(toRemove, 30)] + self.log(f'Found {len(toRemove)} ops to remove in {len(chunks)} chunks') + total = len(chunks) + for i in range(total): + chunk = chunks[i] + self.log(f'Processing chunk {i + 1} of {total}') + self.context.safeDestroyOps(chunk, verbose=True) + await _asyncYield() + + async def _finalizeToolkitPars(self): + comp = self.toolkit + self.context.moveNetworkPane(comp) + comp.par.Devel.readOnly = True + comp.par.externaltox = '' + comp.par.enablecloning = False + comp.par.savebackup = True + if comp.par['reloadtoxonstart'] is not None: + comp.par.reloadtoxonstart = True + else: + comp.par.enableexternaltox = True + comp.par.reloadcustom = True + comp.par.reloadbuiltin = True + focusFirstCustomParameterPage(comp) + + async def _generateToolkitDocs(self): + if self.docProcessor: + self.docProcessor.writeToolkitDocData() + + async def _finishBuild(self): + comp = self.toolkit + self.context.focusInNetworkPane(comp) + toxFile = self._getOutputToxPath('RayTK') + self.log(f'Saving toolkit to {toxFile}') + comp.save(toxFile) + self.log('Finished build') + +class SnippetsBuilderAsync(BuilderAsyncBase): + def __init__(self, context: BuildContext): + super().__init__(context) + self.outputFolder = None # type: Optional[Path] + self.sourceFolder = Path('snippets') + self.tempComp = None # type: Optional[COMP] + self.sourceToxFiles = [] # type: list[Path] + self.opTypesByLocalName = {} # type: dict[str, str] + + async def runBuild(self): + self.log('Starting snippets build') + + await self.logStageStart('Initialization') + await self._initializeOutput() + await self._initializeTempComp() + + await self.logStageStart('Load op table') + await self._loadOpTable() + + await self.logStageStart('Find source tox files') + await self._findSourceToxFiles() + + await self.logStageStart('Process source tox files') + await self._processSourceToxFiles() + + await self.logStageStart('Build zip file') + await self._buildZipFile() + + await self.logStageStart('Clean output') + await self._cleanOutput() + + self.log('Build completed!') + + async def _initializeOutput(self): + tempFolder = Path('build/temp') + tempFolder.mkdir(parents=True, exist_ok=True) + folder = tempFolder / 'snippets' + if folder.exists(): + self.log('Clearing output temp folder ' + folder.as_posix()) + shutil.rmtree(folder) + self.log('Creating output temp folder ' + folder.as_posix()) + folder.mkdir(parents=True) + self.outputFolder = folder + + async def _initializeTempComp(self): + self.tempComp = op('/temp') + if self.tempComp: + self.log('Removing old temp comp') + self.context.safeDestroyOp(self.tempComp) + self.log('Creating temp comp') + self.tempComp = root.create(baseCOMP, 'temp') + + async def _loadOpTable(self): + opTable = RaytkContext().opTable() + self.opTypesByLocalName = {} + for i in range(1, opTable.numRows): + self.opTypesByLocalName[opTable[i, 'name'].val] = opTable[i, 'opType'].val + + async def _findSourceToxFiles(self): + allFiles = (self.sourceFolder / 'operators').rglob('*.tox') + + for file in allFiles: + if file.name == 'index.tox': + continue + if not file.stem.endswith('_snippet'): + continue + opLocalName = file.stem.split('_', maxsplit=1)[0] + if opLocalName not in self.opTypesByLocalName: + self.log('Skipping operator-specific snippet' + file.as_posix()) + continue + self.sourceToxFiles.append(file) + + self.log(f'Found {len(self.sourceToxFiles)} source tox files') + + async def _processSourceToxFiles(self): + for i, tox in enumerate(self.sourceToxFiles): + self.log(f'Processing source tox file [{i+1}/{len(self.sourceToxFiles)}]: {tox}') + await self._processSourceToxFile(tox) + + async def _processSourceToxFile(self, tox: Path): + snippet = self.tempComp.loadTox(tox.as_posix()) + self.log('Processing snippet comp ' + snippet.path) + self.context.focusInNetworkPane(snippet) + self.context.detachTox(snippet) + + await _asyncYield() + rops = RaytkContext().ropChildrenOf(snippet) + self.log(f'Updating {len(rops)} ROPs in snippet') + await _asyncYield() + for rop in rops: + await self._processRop(rop) + + await _asyncYield() + relPath = tox.relative_to(self.sourceFolder) + outputPath = self.outputFolder / relPath + outputPath.parent.mkdir(parents=True, exist_ok=True) + self.log(f'Saving processed snippet to {outputPath}') + snippet.save(outputPath.as_posix()) + + async def _processRop(self, rop: COMP): + self.log('Processing ROP ' + rop.path, verbose=True) + self.context.reclone(rop, verbose=True) + self.context.disableCloning(rop, verbose=True) + self.context.updateROPInstance(rop) + if not rop.isPanel and not rop.isObject: + rop.showCustomOnly = True + + async def _buildZipFile(self): + toxFiles = list(self.outputFolder.rglob('*.tox')) + outputFileName = self._getOutputFilePath('RayTKSnippets', 'zip') + self.log(f'Building zip file {outputFileName} with {len(toxFiles)} files') + with zipfile.ZipFile(outputFileName, 'w') as archive: + for file in toxFiles: + archive.write(file, arcname=file.relative_to(self.outputFolder)) + await _asyncYield() + self.log('Wrote zip file ' + outputFileName) + + async def _cleanOutput(self): + shutil.rmtree(self.outputFolder) + +async def _asyncYield(): + pass diff --git a/devel/build/buildExt.py b/devel/build/buildExt.py deleted file mode 100644 index 2c304e0c2..000000000 --- a/devel/build/buildExt.py +++ /dev/null @@ -1,828 +0,0 @@ -from datetime import datetime -from pathlib import Path -import shutil -import zipfile - -from raytkTools import RaytkTools -from raytkUtil import RaytkTags, navigateTo, focusFirstCustomParameterPage, CategoryInfo, RaytkContext, IconColors -from raytkBuild import BuildContext, DocProcessor, chunked_iterable -from typing import Callable, Optional, TextIO - -# noinspection PyUnreachableCode -if False: - # noinspection PyUnresolvedReferences - from _stubs import * - -class BuildManager: - def __init__(self, ownerComp: COMP): - self.ownerComp = ownerComp - self.logTable = ownerComp.op('log') - self.context = None # type: BuildContext | None - self.docProcessor = None # type: DocProcessor | None - self.experimentalMode = False - self.logFile = None # type: TextIO | None - self.enableVerboseLogging = False - - def OnInit(self): - self.ClearLog() - - @staticmethod - def GetToolkitVersion(): - return RaytkContext().toolkitVersion() - - @staticmethod - def OpenToolkitNetwork(): - navigateTo(RaytkContext().toolkit(), name='raytkBuildNetwork', popup=True, goInto=True) - - def OpenLog(self): - dat = self.ownerComp.op('full_log_text') - dat.openViewer() - - def ClearLog(self): - self.logTable.clear() - self.closeLogFile() - - def closeLogFile(self): - if not self.logFile: - return - self.logFile.close() - self.logFile = None - - def startNewLogFile(self): - stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') - fileName = f'build/log/build-{stamp}.txt' - filePath = Path(fileName) - filePath.parent.mkdir(parents=True, exist_ok=True) - self.logFile = filePath.open('a') - - def ReloadToolkit(self): - self.logTable.clear() - self.log('Reloading toolkit') - toolkit = RaytkContext().toolkit() - queueCall(self.reloadToolkit, toolkit) - - def ReloadSnippets(self): - self.logTable.clear() - self.log('Reloading snippets') - snippets = getattr(op, 'raytkSnippets') - queueCall(self.reloadSnippets, snippets) - - def prepareForBuild(self): - self.experimentalMode = bool(self.ownerComp.op('experimental_toggle').par.Value0) - self.enableVerboseLogging = bool(self.ownerComp.op('verboseLogging_toggle').par.Value0) - self.logTable.clear() - self.closeLogFile() - if self.ownerComp.op('useLogFile_toggle').par.Value0: - self.startNewLogFile() - self.context = BuildContext(self.log, self.experimentalMode) - - def runToolkitBuildOnly(self): - self.prepareForBuild() - def afterBuild(): - self.log('Finished toolkit build process') - self.closeLogFile() - builder = ToolkitBuilder( - self.context, - reloadToolkit=self.reloadToolkit, - ) - builder.runBuild(thenRun=afterBuild) - - def runSnippetBuildOnly(self): - self.prepareForBuild() - def afterBuild(): - self.log('Finished snippets build process') - self.closeLogFile() - builder = SnippetsBuilder( - self.context, - reloadSnippets=self.reloadSnippets, - ) - builder.runBuild(thenRun=afterBuild) - - def runSeparateSnippetBuildOnly(self): - self.prepareForBuild() - def afterBuild(): - self.log('Finished snippets build process') - self.closeLogFile() - builder = SeparateSnippetsBuilder( - self.context, - ) - builder.runBuild(thenRun=afterBuild) - - def runToolkitAndSnippetBuilds(self): - self.prepareForBuild() - def afterBuild(): - self.log('Finished full build process') - self.closeLogFile() - def afterToolkitBuild(): - self.log('After toolkit has built, Processing snippets...') - snippetsBuilder = SnippetsBuilder( - self.context, - reloadSnippets=self.reloadSnippets, - ) - snippetsBuilder.runBuild(thenRun=afterBuild) - builder = ToolkitBuilder( - self.context, - reloadToolkit=self.reloadToolkit, - ) - builder.runBuild(thenRun=afterToolkitBuild) - - @staticmethod - def reloadToolkit(toolkit: COMP): - toolkit.par.externaltox = 'src/raytk.tox' - toolkit.par.reinitnet.pulse() - # Do this early since it switches off things like automatically writing to the opList.txt file. - # See https://github.com/t3kt/raytk/issues/95 - toolkit.par.Devel = False - - @staticmethod - def reloadSnippets(snippets: COMP): - snippets.par.externaltox = 'snippets/raytkSnippets.tox' - snippets.par.reinitnet.pulse() - # Do this early since it switches off things like automatically writing to the opList.txt file. - # See https://github.com/t3kt/raytk/issues/95 - snippets.par.Devel = False - - def log(self, message: str, verbose=False): - if verbose and not self.enableVerboseLogging: - return - print(message, flush=True) - if self.logFile: - stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') - print(stamp, message, file=self.logFile, flush=True) - self.logTable.appendRow([message]) - - @staticmethod - def queueMethodCall(method: callable, *args): - queueCall(method, *args) - -class _BuilderBase: - def __init__( - self, - context: BuildContext, - ): - self.context = context - self.afterBuild = None # type: Callable | None - - def runBuild(self, thenRun: Callable): - self.afterBuild = thenRun - pass - - def removeBuildExcludeOpsIn(self, scope: COMP, thenRun: callable): - self.log(f'Removing buildExclude ops in {scope} (deep)') - toRemove = scope.findChildren(tags=[RaytkTags.buildExclude.name], includeUtility=True) - chunks = [list(chunk) for chunk in chunked_iterable(toRemove, 30)] - self.log(f'Found {len(toRemove)} ops to remove in {len(chunks)} chunks') - total = len(chunks) - def runPart(): - if not chunks: - queueCall(thenRun) - else: - chunk = chunks.pop() - self.log(f'Processing chunk {total - len(chunks)} / {total}') - self.context.safeDestroyOps(chunk, verbose=True) - queueCall(runPart) - runPart() - - def getOutputToxPath(self, baseName: str): - return self.getOutputFilePath(baseName, 'tox') - - def getOutputFilePath(self, baseName: str, extension: str): - version = RaytkContext().toolkitVersion() - if self.context.experimental: - suffix = '-exp' - else: - suffix = '' - return f'build/{baseName}-{version}{suffix}.{extension}' - - def finalizeRootPars(self, comp: COMP): - self.context.moveNetworkPane(comp) - comp.par.Devel.readOnly = True - comp.par.externaltox = '' - comp.par.enablecloning = False - comp.par.savebackup = True - if comp.par['reloadtoxonstart'] is not None: - comp.par.reloadtoxonstart = True - else: - comp.par.enableexternaltox = True - comp.par.reloadcustom = True - comp.par.reloadbuiltin = True - focusFirstCustomParameterPage(comp) - - def log(self, message: str, verbose=False): - self.context.log(message, verbose) - - def logStageStart(self, desc: str): - self.log(f'===[{desc}]===') - -class ToolkitBuilder(_BuilderBase): - def __init__( - self, - context: BuildContext, - reloadToolkit: Callable, - ): - super().__init__(context) - self.reloadToolkit = reloadToolkit - self.docProcessor = None # type: DocProcessor | None - if not self.context.experimental: - self.docProcessor = DocProcessor( - self.context, - dataFolder='docs/_data', - referenceFolder='docs/_reference', - imagesFolder='docs/assets/images', - ) - - def runBuild(self, thenRun: callable): - super().runBuild(thenRun) - version = RaytkContext().toolkitVersion() - self.log('Starting build') - self.log(f'Version: {version}' + (' (experimental)' if self.context.experimental else '')) - queueCall(self.runBuild_stage, 0) - - def runBuild_stage(self, stage: int): - toolkit = RaytkContext().toolkit() - if stage == 0: - self.log('Disabling snippets') - snippets = getattr(op, 'raytkSnippets', None) - if snippets: - snippets.allowCooking = False - # self.log('NOT ************ Reloading toolkit') - # WARNING: SKIPPING RELOAD!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - self.log('Reloading toolkit') - self.reloadToolkit(toolkit) - self.context.openNetworkPane() - queueCall(self.runBuild_stage, stage + 1) - elif stage == 1: - self.logStageStart('Detach fileSync ops') - self.context.detachAllFileSyncDatsIn(toolkit, reloadFirst=True) - queueCall(self.runBuild_stage, stage + 1) - elif stage == 2: - if self.docProcessor: - self.logStageStart('Clear old docs') - self.docProcessor.clearPreviousDocs() - queueCall(self.runBuild_stage, stage + 1) - elif stage == 3: - self.logStageStart('Process thumbnails') - self.context.runBuildScript( - toolkit.op('libraryThumbs/BUILD'), - thenRun=self.runBuild_stage, runArgs=[stage + 1]) - elif stage == 4: - self.logStageStart('Update library info') - self.updateLibraryInfo(toolkit, thenRun=self.runBuild_stage, runArgs=[stage + 1]) - elif stage == 5: - self.logStageStart('Update library image') - self.updateLibraryImage(toolkit, thenRun=self.runBuild_stage, runArgs=[stage + 1]) - elif stage == 6: - self.logStageStart('Preprocessing components') - self.preProcessComponents(toolkit.op('components'), thenRun=self.runBuild_stage, runArgs=[stage + 1]) - elif stage == 7: - self.logStageStart('Process operators') - self.processOperators(toolkit.op('operators'), thenRun=self.runBuild_stage, runArgs=[stage + 1]) - elif stage == 8: - self.logStageStart('Process nested operators') - self.processNestedOperators(toolkit.op('operators'), thenRun=self.runBuild_stage, runArgs=[stage + 1]) - elif stage == 9: - self.logStageStart('Lock library info') - self.lockLibraryInfo(toolkit, thenRun=self.runBuild_stage, runArgs=[stage + 1]) - elif stage == 10: - self.logStageStart('Remove op help') - self.removeAllOpHelp(thenRun=self.runBuild_stage, runArgs=[stage + 1]) - elif stage == 11: - self.logStageStart('Process tools') - self.processTools(toolkit.op('tools'), thenRun=self.runBuild_stage, runArgs=[stage + 1]) - elif stage == 12: - self.logStageStart('Lock buildLock ops') - self.context.lockBuildLockOps(toolkit) - queueCall(self.runBuild_stage, stage + 1) - elif stage == 13: - self.logStageStart('Process components') - self.processComponents(toolkit.op('components'), thenRun=self.runBuild_stage, runArgs=[stage + 1]) - elif stage == 14: - self.logStageStart('Remove buildExclude ops') - self.removeBuildExcludeOpsIn(toolkit, thenRun=lambda: self.runBuild_stage(stage + 1)) - elif stage == 15: - self.logStageStart('Remove redundant python mods') - self.context.removeRedundantPythonModules(toolkit, toolkit.ops('tools', 'libraryInfo')) - queueCall(self.runBuild_stage, stage + 1) - elif stage == 16: - self.logStageStart('Finalize toolkit pars') - self.finalizeRootPars(toolkit) - queueCall(self.runBuild_stage, stage + 1) - elif stage == 17: - self.logStageStart('Write toolkit doc data') - if self.docProcessor: - self.docProcessor.writeToolkitDocData() - queueCall(self.runBuild_stage, stage + 1) - elif stage == 18: - self.logStageStart('Finish build') - self.context.focusInNetworkPane(toolkit) - toxFile = self.getOutputToxPath('RayTK') - self.log(f'Exporting TOX to {toxFile}') - toolkit.save(toxFile) - self.log('Build completed!') - self.log(f'Exported tox file: {toxFile}') - queueCall(self.afterBuild) - - def updateLibraryImage( - self, toolkit: COMP, - thenRun: 'Optional[Callable]' = None, runArgs: list = None): - self.log('Updating library image') - image = RaytkContext().libraryImage() - if image: - # self.context.moveNetworkPane(image) - self.context.detachTox(image) - self.context.lockBuildLockOps(image) - image.par.Showshortcut = True - toolkit.par.opviewer.val = image - toolkit.par.opviewer.readOnly = True - toolkit.viewer = True - else: - toolkit.par.opviewer.val = '' - toolkit.viewer = False - if thenRun: - queueCall(thenRun, *(runArgs or [])) - - def updateLibraryInfo( - self, toolkit: COMP, - thenRun: 'Optional[Callable]' = None, runArgs: list = None): - self.log('Updating library info') - if toolkit.par['Experimentalbuild'] is not None: - toolkit.par.Experimentalbuild.val = self.context.experimental - toolkit.par.Experimentalbuild.readOnly = True - libraryInfo = toolkit.op('libraryInfo') - libraryInfo.par.Forcebuild.pulse() - self.context.moveNetworkPane(libraryInfo) - self.context.runBuildScript( - libraryInfo.op('BUILD'), - thenRun=lambda: queueCall(thenRun, *(runArgs or [])), - runArgs=[]) - - def lockLibraryInfo( - self, toolkit: COMP, - thenRun: 'Optional[Callable]' = None, runArgs: list = None): - self.log('Locking library info') - self.context.lockOps(toolkit.ops( - 'info', 'opTable', 'opCategoryTable', 'opHelpTable', 'buildInfo')) - if thenRun: - queueCall(thenRun, *(runArgs or [])) - - def preProcessComponents( - self, components: COMP, - thenRun: 'Optional[Callable]' = None, runArgs: list = None): - self.log(f'Prepocessing components {components}') - self.context.focusInNetworkPane(components) - shared = components.op('shared') - self.context.lockBuildLockOps(shared) - # this one has a buildLock typeTable that isn't instance-specific, - # so we lock it first then delete the stuff that built it - typeSpec = components.op('typeSpec') - self.context.lockBuildLockOps(typeSpec) - opDef = components.op('opDefinition') - self.context.lockBuildLockOps(opDef) - comps = components.ops( - 'typeSpec', 'typeResolver', 'typeRestrictor', - # 'opImage', # this has some instance-dependent stuff - 'opDefinition', 'compDefinition', - # 'inputHandler', # don't process inputHandler since it uses buildExclude for the config tables, which - # won't have been locked yet when this stage runs - 'shaderBuilder', - 'opElement', 'transformCodeGenerator', 'timeProvider', - 'supportDetector', 'expresssionSwitcher', 'parMenuUpdater', - # 'codeSwitcher', - 'aggregateCodeGenerator', - # 'combiner', # don't process combiner since it has instance-specific buildLock things depending - # on buildExclude - 'waveFunction', - ) - def nextStage(): - if not comps: - queueCall(thenRun, *(runArgs or [])) - else: - o = comps.pop() - self.removeBuildExcludeOpsIn(o, thenRun=nextStage) - queueCall(nextStage) - - def processComponents( - self, components: COMP, - thenRun: 'Optional[Callable]' = None, runArgs: list = None): - self.log(f'Processing components {components}') - self.context.focusInNetworkPane(components) - self.context.safeDestroyOp(components) - queueCall(thenRun, *(runArgs or [])) - - def processOperators( - self, comp: COMP, - thenRun: 'Optional[Callable]' = None, runArgs: list = None): - self.log(f'Processing operators {comp}') - self.context.moveNetworkPane(comp) - self.context.detachTox(comp) - categories = RaytkContext().allCategories() - # categories.sort(key=lambda c: c.name) - if self.docProcessor: - self.docProcessor.writeCategoryListPage(categories) - queueCall(self.processOperatorCategories_stage, categories, thenRun, runArgs) - - def processOperatorCategories_stage( - self, categories: list[COMP], - thenRun: 'Optional[Callable]' = None, runArgs: list = None): - if categories: - category = categories.pop() - self.processOperatorCategory( - category, - thenRun=self.processOperatorCategories_stage, - runArgs=[categories, thenRun, runArgs]) - elif thenRun: - queueCall(thenRun, *(runArgs or [])) - - def processOperatorCategory( - self, category: COMP, - thenRun: 'Optional[Callable]' = None, runArgs: list = None): - categoryInfo = CategoryInfo(category) - self.log(f'Processing operator category {category.name}') - self.context.moveNetworkPane(category) - self.context.detachTox(category) - template = category.op('__template') - if template: - template.destroy() - if not self.context.experimental: - self.context.removeAlphaOps(category) - comps = categoryInfo.operators - # comps.sort(key=lambda c: c.name) - queueCall(self.processOperatorCategory_stage, category, comps, thenRun, runArgs) - - def processOperatorCategory_stage( - self, category: COMP, components: list[COMP], - thenRun: 'Optional[Callable]' = None, runArgs: list = None): - if components: - comp = components.pop() - self.processOperator(comp) - queueCall(self.processOperatorCategory_stage, category, components, thenRun, runArgs) - else: - # after finishing the operators in the category, process the category help - if self.docProcessor: - self.docProcessor.processOpCategory(category) - self.context.removeCatHelp(category) - if thenRun: - queueCall(thenRun, *(runArgs or [])) - - def processOperator(self, comp: COMP): - self.log(f'Processing operator {comp}') - self.context.focusInNetworkPane(comp) - self.context.disableCloning(comp) - self.context.detachTox(comp) - tools = RaytkTools() - tools.updateROPMetadata(comp) - tools.updateROPParams(comp) - self.context.applyParamUpdatersIn(comp) - self.context.resetCustomPars(comp) - self.context.lockROPPars(comp) - - # This really shouldn't be necessary but there's something strange with old cloned components... - self.context.safeDestroyOp(comp.op('opDefinition/paramHelpEditor')) - self.context.safeDestroyOp(comp.op('shaderBuilder/supportDetector')) - - # self.context.moveNetworkPane(comp) - self.processOperatorSubCompChildrenOf(comp) - # self.context.moveNetworkPane(comp) - if not comp.isPanel: - comp.showCustomOnly = True - self.log(f'Updating OP image for {comp}') - img = tools.updateOPImage(comp) - if img: - # self.context.focusInNetworkPane(img) - self.context.disableCloning(img) - self.context.detachTox(img) - self.context.lockBuildLockOps(img) - self.context.cleanOpImage(img) - comp.color = IconColors.defaultBgColor - if self.docProcessor: - self.docProcessor.processOp(comp) - - def processOperatorSubCompChildrenOf(self, comp: COMP): - subComps = comp.findChildren(type=COMP) - if not subComps: - return - self.context.log(f'Processing {len(subComps)} sub-comps in {comp}', verbose=True) - for child in subComps: - self.processOperatorSubComp_2(child) - - def processOperatorSubComp_2(self, comp: COMP): - self.context.log(f'Processing {comp}', verbose=True) - self.context.detachTox(comp) - self.context.reclone(comp) - self.context.disableCloning(comp) - self.processOperatorSubCompChildrenOf(comp) - - def processNestedOperators( - self, comp: COMP, - thenRun: 'Optional[Callable]' = None, runArgs: list = None): - self.log('Processing nested operators') - subOps = comp.findChildren(tags=[RaytkTags.raytkOP.name], depth=3) - self.log(f'found {len(subOps)} nested operators') - queueCall(self.processNestedOperators_stage, subOps, thenRun, runArgs) - - def processNestedOperators_stage( - self, comps: list[COMP], - thenRun: 'Optional[Callable]' = None, runArgs: list = None): - if comps: - comp = comps.pop() - self.processNestedOperator(comp) - if comps: - queueCall(self.processNestedOperators_stage, comps, thenRun, runArgs) - return - if thenRun: - queueCall(thenRun, *(runArgs or [])) - - def processNestedOperator(self, rop: COMP): - self.log(f'Processing sub-operator {rop.path}') - self.context.updateOrReclone(rop) - self.context.detachTox(rop) - self.context.disableCloning(rop) - - def removeAllOpHelp( - self, - thenRun: 'Optional[Callable]' = None, runArgs: list = None): - operators = RaytkContext().allMasterOperators() - for comp in operators: - self.context.removeOpHelp(comp) - queueCall(thenRun, *(runArgs or [])) - - def processTools( - self, comp: COMP, - thenRun: 'Optional[Callable]' = None, runArgs: list = None): - self.log(f'Processing tools {comp}') - self.context.moveNetworkPane(comp) - self.context.reloadTox(comp) - self.context.detachTox(comp) - self.context.runBuildScript( - comp.op('BUILD'), - thenRun=lambda: queueCall(thenRun, *(runArgs or [])), - runArgs=[]) - -class SnippetsBuilder(_BuilderBase): - def __init__( - self, - context: BuildContext, - reloadSnippets: Callable, - ): - super().__init__(context) - self.reloadSnippets = reloadSnippets - - def runBuild(self, thenRun: Callable): - super().runBuild(thenRun) - self.log('Starting snippets build') - queueCall(self.runBuild_stage, 0) - - def runBuild_stage(self, stage: int): - snippets = getattr(op, 'raytkSnippets', None) - if not snippets: - self.context.log('ERROR: Snippets not found!') - raise Exception('Snippets not found!') - if stage == 0: - self.log('Reloading snippets root') - self.reloadSnippets(snippets) - if not snippets.allowCooking: - self.log('Enabling cooking for snippets root') - snippets.allowCooking = True - queueCall(self.runBuild_stage, stage + 1) - elif stage == 1: - self.logStageStart('Process navigator') - self.context.runBuildScript(snippets.op('navigator/BUILD'), thenRun=self.runBuild_stage, runArgs=[stage + 1]) - elif stage == 2: - self.logStageStart('Process snippets structure') - self.processSnippetsStructure(snippets) - queueCall(self.runBuild_stage, stage + 1) - elif stage == 3: - self.logStageStart('Process snippets') - self.processSnippets(snippets, thenRun=self.runBuild_stage, runArgs=[stage + 1]) - elif stage == 4: - self.logStageStart('Remove buildExclude ops') - self.removeBuildExcludeOpsIn(snippets, thenRun=lambda: self.runBuild_stage(stage + 1)) - elif stage == 5: - self.logStageStart('Finalize snippets root pars') - self.finalizeRootPars(snippets) - queueCall(self.runBuild_stage, stage + 1) - elif stage == 6: - self.logStageStart('Finish snippets build') - self.context.focusInNetworkPane(snippets) - toxFile = self.getOutputToxPath('RayTKSnippets') - self.log(f'Exporting TOX to {toxFile}') - snippets.save(toxFile) - self.log('Build completed!') - self.log(f'Exported tox file: {toxFile}') - - def processSnippetsStructure(self, snippets: COMP): - snippetsRoot = snippets.op('snippets') - self.context.detachTox(snippetsRoot) - operatorsRoot = snippets.op('snippets/operators') - self.context.detachTox(operatorsRoot) - for comp in operatorsRoot.children: - if not comp.isCOMP: - continue - self.context.detachTox(comp) - comp.allowCooking = True - operatorsRoot.allowCooking = True - snippetsRoot.allowCooking = True - - def processSnippets(self, snippets: COMP, thenRun: Callable, runArgs: list): - self.log('Processing snippets') - snippetsRoot = snippets.op('snippets') - snippetTable = snippets.op('navigator/snippetTable') # type: DAT - opTable = RaytkContext().opTable() - knownOpTypes = [c.val for c in opTable.col('opType')[1:]] - - self.log(f'Found {snippetTable.numRows - 1} snippets') - - def processSnippetsStage(row: int): - if row >= snippetTable.numRows: - queueCall(thenRun, *runArgs) - else: - snippetOpType = snippetTable[row, 'opType'].val - snippet = snippetsRoot.op(snippetTable[row, 'relPath']) - self.log(f'Processing snippet {snippet}') - self.context.focusInNetworkPane(snippet) - if snippetOpType not in knownOpTypes: - self.log(f'Removing snippet for missing type {snippetOpType}') - self.context.safeDestroyOp(snippet) - queueCall(processSnippetsStage, row + 1) - else: - queueCall(self.processSnippet, snippet, processSnippetsStage, [row + 1]) - - queueCall(processSnippetsStage, 1) - - def processSnippet(self, snippet: COMP, theRun: Callable, runArgs: list): - self.context.detachTox(snippet) - - def processSnippetStage(stage: int): - if stage == 0: - self.log('Enabling snippet cooking') - snippet.allowCooking = True - elif stage == 1: - rops = RaytkContext().ropChildrenOf(snippet) - self.log(f'Updating {len(rops)} ROPs') - for rop in rops: - self.processRop(rop) - elif stage == 2: - self.log('Disabling snippet cooking') - snippet.allowCooking = False - else: - queueCall(theRun, *(runArgs or [])) - return - queueCall(processSnippetStage, stage + 1) - - queueCall(processSnippetStage, 0) - - def processRop(self, rop: COMP): - self.log(f'Processing ROP {rop}', verbose=True) - rop.par.enablecloningpulse.pulse() - # self.context.reclone(rop, verbose=True) - rop.par.enablecloning = False - if not rop.isPanel and not rop.isObject: - rop.showCustomOnly = True - - def finalizeRootPars(self, comp: COMP): - super().finalizeRootPars(comp) - version = RaytkContext().toolkitVersion() - comp.par.Raytkversion = version - comp.par.Raytkversion.default = version - comp.par.Experimentalbuild = self.context.experimental - comp.par.Experimentalbuild.default = self.context.experimental - -class SeparateSnippetsBuilder(_BuilderBase): - def __init__( - self, - context: BuildContext, - ): - super().__init__(context) - self.outputFolder = None # type: Optional[Path] - self.sourceFolder = Path('snippets') - self.tempComp = None # type: Optional[COMP] - self.sourceToxFiles = [] # type: list[Path] - self.opTypesByLocalName = {} # type: dict[str, str] - - def runBuild(self, thenRun: Callable): - super().runBuild(thenRun) - self.log('Starting separated snippets build') - queueCall(self.runBuild_stage, 0) - - def runBuild_stage(self, stage: int): - if stage == 0: - self.logStageStart('Initialization') - self.initializeOutput() - self.initializeTempComp() - queueCall(self.runBuild_stage, stage + 1) - elif stage == 1: - self.findSourceToxFiles() - queueCall(self.runBuild_stage, stage + 1) - elif stage == 2: - self.processSourceToxFiles(thenRun=lambda: self.runBuild_stage(stage + 1)) - elif stage == 3: - outputFileName = self.buildZip() - self.log(f'Exported zip file: {outputFileName}') - queueCall(self.runBuild_stage, stage + 1) - elif stage == 4: - self.cleanOutput() - queueCall(self.runBuild_stage, stage + 1) - elif stage == 5: - self.log('Build completed!') - queueCall(self.afterBuild) - - def initializeOutput(self): - tempFolder = Path('build/temp') - tempFolder.mkdir(parents=True, exist_ok=True) - folder = tempFolder / 'snippets' - if folder.exists(): - self.log('Clearing output temp folder ' + folder.as_posix()) - shutil.rmtree(folder) - self.log('Creating output temp folder ' + folder.as_posix()) - folder.mkdir(parents=True) - self.outputFolder = folder - - def initializeTempComp(self): - self.tempComp = op('/temp') - if self.tempComp: - self.log('Removing old temp comp') - self.context.safeDestroyOp(self.tempComp) - self.log('Creating temp comp') - self.tempComp = root.create(baseCOMP, 'temp') - - def parseOpTable(self): - self.log('Parsing op table') - opTable = RaytkContext().opTable() - self.opTypesByLocalName = {} - for i in range(1, opTable.numRows): - self.opTypesByLocalName[opTable[i, 'name'].val] = opTable[i, 'opType'].val - - def findSourceToxFiles(self): - self.logStageStart('Find source tox files') - - def processFiles(files, isOperatorSpecific=False): - for file in files: - if file.name == 'index.tox': - continue - if isOperatorSpecific: - if file.name.endswith('_snippet') and file.name.split('_', maxsplit=1)[0] not in self.opTypesByLocalName: - self.log(f'Skipping operator-specific snippet {file}') - continue - self.sourceToxFiles.append(file) - - processFiles((self.sourceFolder / 'operators').rglob('*.tox'), isOperatorSpecific=True) - # TODO: other root folders? - self.log(f'Found {len(self.sourceToxFiles)} source tox files') - - def processSourceToxFiles(self, thenRun: callable): - self.logStageStart('Process source tox files') - self.processSourceToxFiles_stage(0, thenRun) - - def processSourceToxFiles_stage(self, i: int, thenRun: callable): - if i >= len(self.sourceToxFiles): - self.log('Finished processing source tox files') - queueCall(thenRun) - else: - tox = self.sourceToxFiles[i] - self.log(f'Processing source tox file [{ i + 1 } / {len(self.sourceToxFiles)}] {tox}') - self.processSourceToxFile(tox, thenRun=lambda: self.processSourceToxFiles_stage(i + 1, thenRun)) - - def processSourceToxFile(self, tox: Path, thenRun: callable): - snippet = self.tempComp.loadTox(tox.as_posix()) - self.log('Processing snippet comp ' + snippet.path) - self.context.focusInNetworkPane(snippet) - self.context.detachTox(snippet) - def processStage1(): - rops = RaytkContext().ropChildrenOf(snippet) - self.log(f'Updating {len(rops)} ROPs') - for rop in rops: - self.processRop(rop) - queueCall(processStage2) - def processStage2(): - relPath = tox.relative_to(self.sourceFolder) - outputPath = self.outputFolder / relPath - outputPath.parent.mkdir(parents=True, exist_ok=True) - self.log(f'Writing tox to {outputPath}') - snippet.save(outputPath.as_posix()) - queueCall(thenRun) - queueCall(processStage1) - - def processRop(self, rop: COMP): - self.log(f'Processing ROP {rop}', verbose=True) - rop.par.enablecloningpulse.pulse() - # self.context.reclone(rop, verbose=True) - rop.par.enablecloning = False - if not rop.isPanel and not rop.isObject: - rop.showCustomOnly = True - - def buildZip(self): - self.logStageStart('Finish snippets build') - toxFiles = list(self.outputFolder.rglob('*.tox')) - outputFileName = self.getOutputFilePath('RayTKSnippets', 'zip') - self.log(f'Building {outputFileName} with {len(toxFiles)} tox files') - with zipfile.ZipFile(outputFileName, 'w') as archive: - for file in toxFiles: - archive.write(file, arcname=file.relative_to(self.outputFolder)) - return outputFileName - - def cleanOutput(self): - self.logStageStart('Cleaning temporary output') - shutil.rmtree(self.outputFolder) - -def queueCall(action: Callable, *args): - run('args[0](*(args[1:]))', action, *args, delayFrames=10, delayRef=root) diff --git a/devel/components/misc/misc.tox b/devel/components/misc/misc.tox index bf0f39294..1eb6f164d 100644 Binary files a/devel/components/misc/misc.tox and b/devel/components/misc/misc.tox differ diff --git a/devel/devel.tox b/devel/devel.tox index 807c7c21f..21b541a76 100644 Binary files a/devel/devel.tox and b/devel/devel.tox differ diff --git a/devel/streamer/streamer.tox b/devel/streamer/streamer.tox new file mode 100644 index 000000000..c9dd347ff Binary files /dev/null and b/devel/streamer/streamer.tox differ diff --git a/devel/tester/tester.py b/devel/tester/tester.py index f89c6d8f8..301443ee6 100644 --- a/devel/tester/tester.py +++ b/devel/tester/tester.py @@ -2,6 +2,7 @@ from pathlib import Path from raytkTest import TestCaseResult, TestFindingStatus, processTest from raytkUtil import RaytkContext, recloneComp, Version +import re # noinspection PyUnreachableCode if False: @@ -111,20 +112,25 @@ def buildToolkitVersionTable(self, dat: DAT, fileTable: DAT): dat.appendRow(['name', 'label', 'version', 'tox']) dat.appendRow(['src', 'Current Source', '(current)', 'src/raytk.tox']) buildFolder = Path(self.ownerComp.par.Buildfolder.eval()) + pattern = re.compile(r'^RayTK-(\d+\.?\d+)(-exp(-\d+)?)?$') for row in range(1, fileTable.numRows): relFile = Path(str(fileTable[row, 'relpath'])) toxFile = buildFolder / relFile - versionStr = toxFile.stem - isExp = versionStr.endswith('-exp') - if isExp: - versionStr = versionStr.replace('-exp', '') - if '-' in versionStr: - versionStr = versionStr.split('-', 1)[1] + match = pattern.match(toxFile.stem) + if not match: + debug(f' no match for {toxFile.stem}') + continue + versionStr = match.group(1) + isExp = bool(match.group(2)) + expIterationSuffix = match.group(3) try: version = Version(versionStr) except ValueError: version = None - expSuffix = ' (exp)' if isExp else '' + if not isExp: + expSuffix = '' + else: + expSuffix = ' (exp' + (expIterationSuffix or '') + ')' dat.appendRow([ toxFile.stem, f'Build {version}{expSuffix}' if version else f'{toxFile.stem}{expSuffix}', diff --git a/devel/thirdParty/TDAsyncIO.py b/devel/thirdParty/TDAsyncIO.py new file mode 100644 index 000000000..41aa3f623 --- /dev/null +++ b/devel/thirdParty/TDAsyncIO.py @@ -0,0 +1,73 @@ +""" +TDAsyncIO - Utilities for asyncio library with TouchDesigner + +Copyright (C) 2021 Motoki Sonoda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +# noinspection PyUnreachableCode +if False: + # noinspection PyUnresolvedReferences + from _stubs import * + +import asyncio +# from TDStoreTools import StorageManager +# TDF = op.TDModules.mod.TDFunctions +import sys +from typing import Coroutine + +class TDAsyncIO: + """ + TDAsyncIO description + """ + def __init__(self, ownerComp): + # The component to which this extension is attached + self.ownerComp = ownerComp + + # Get the current event loop. + self.loop = asyncio.get_event_loop() + + # If the current event loop was closed, then create a new event loop. + if self.loop.is_closed(): + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + + def __delTD__(self): + # Check this component is global OP or not. + if me.parent() == op.TDAsyncIO: + # Close the event loop. The loop must not be running. + # Pending callbacks will be lost. + self.loop.close() + + def Run(self, coroutines: list[Coroutine]): + for coroutine in coroutines: + self.loop.create_task(coroutine) + + def Update(self): + self.loop.call_soon(self.loop.stop) + self.loop.run_forever() + + def Cancel(self): + if sys.version_info[0] >= 3 and sys.version_info[1] >=7: + for task in asyncio.all_tasks(self.loop): + task.cancel() + else: + for task in asyncio.Task.all_tasks(self.loop): + task.cancel() diff --git a/devel/thirdParty/TDAsyncIO.tox b/devel/thirdParty/TDAsyncIO.tox new file mode 100644 index 000000000..632d7f753 Binary files /dev/null and b/devel/thirdParty/TDAsyncIO.tox differ diff --git a/devel/toolkitEditor/createRopDialog/createRopDialog.tox b/devel/toolkitEditor/createRopDialog/createRopDialog.tox index 09415813c..88b3ce82c 100644 Binary files a/devel/toolkitEditor/createRopDialog/createRopDialog.tox and b/devel/toolkitEditor/createRopDialog/createRopDialog.tox differ diff --git a/devel/toolkitEditor/mainMenu/mainMenu.py b/devel/toolkitEditor/mainMenu/mainMenu.py index e34cf29a7..32cbdac08 100644 --- a/devel/toolkitEditor/mainMenu/mainMenu.py +++ b/devel/toolkitEditor/mainMenu/mainMenu.py @@ -88,7 +88,13 @@ def __init__(self, ownerComp: COMP): 'Save All ROP Specs', menuName='Tools', action=lambda: ext.toolkitEditor.saveAllROPSpecs(), - ) + ), + _MenuItem( + 'saveIncrementAllRops', + 'Save and Increment All ROPs', + menuName='Tools', + action=lambda: ext.toolkitEditor.saveAllROPs(incrementVersion=True), + ), ], } diff --git a/devel/toolkitEditor/mainMenu/mainMenu.tox b/devel/toolkitEditor/mainMenu/mainMenu.tox index a6a55573d..e36761639 100644 Binary files a/devel/toolkitEditor/mainMenu/mainMenu.tox and b/devel/toolkitEditor/mainMenu/mainMenu.tox differ diff --git a/devel/toolkitEditor/ropEditor/elementsPanel/buildItemTable.py b/devel/toolkitEditor/ropEditor/elementsPanel/buildItemTable.py index fac8df9dd..b62cd2989 100644 --- a/devel/toolkitEditor/ropEditor/elementsPanel/buildItemTable.py +++ b/devel/toolkitEditor/ropEditor/elementsPanel/buildItemTable.py @@ -1,2 +1,2 @@ def onCook(dat): - ext.elementsPanel.buildItemTable(dat, dat.inputs[0]) + ext.elementsPanel.buildItemTable(dat) diff --git a/devel/toolkitEditor/ropEditor/elementsPanel/elementsPanel.py b/devel/toolkitEditor/ropEditor/elementsPanel/elementsPanel.py index 587b23e17..dbdf6bb0b 100644 --- a/devel/toolkitEditor/ropEditor/elementsPanel/elementsPanel.py +++ b/devel/toolkitEditor/ropEditor/elementsPanel/elementsPanel.py @@ -1,8 +1,17 @@ +from raytkUtil import ROPInfo + # noinspection PyUnreachableCode if False: # noinspection PyUnresolvedReferences from _stubs import * from _typeAliases import * + from _typeAliases import * + + class _InspectorCorePars(ParCollection): + Hastarget: 'BoolParamT' + Definitiontable: 'DatParamT' + Targetcomp: 'CompParamT' + ipar.inspectorCore = _InspectorCorePars() class _StatePar(ParCollection): XYZ: 'BoolParamT' @@ -13,11 +22,18 @@ def __init__(self, ownerComp: COMP): self.ownerComp = ownerComp @staticmethod - def buildItemTable(dat: DAT, elementTable: DAT): + def buildItemTable(dat: DAT): dat.clear() dat.appendRow(['name', 'label', 'hostOp', 'parName', 'datName', 'evalDatName', 'fileSuffix']) - for row in range(1, elementTable.numRows): - elementRoot = op(elementTable[row, 'elementRoot']) + rop = ipar.inspectorCore.Targetcomp.eval() + info = ROPInfo(rop) + if not info.isROP: + return + state = info.opDefExt.getRopState() + if not state or not state.opElements: + return + for element in state.opElements: + elementRoot = op(element.elementRoot) if not elementRoot: continue for par in elementRoot.customPars: diff --git a/devel/toolkitEditor/ropEditor/elementsPanel/elementsPanel.tox b/devel/toolkitEditor/ropEditor/elementsPanel/elementsPanel.tox index 5cbb5074d..dd7142a0f 100644 Binary files a/devel/toolkitEditor/ropEditor/elementsPanel/elementsPanel.tox and b/devel/toolkitEditor/ropEditor/elementsPanel/elementsPanel.tox differ diff --git a/devel/toolkitEditor/ropEditor/ropEditor.tox b/devel/toolkitEditor/ropEditor/ropEditor.tox index 327232af9..877df0837 100644 Binary files a/devel/toolkitEditor/ropEditor/ropEditor.tox and b/devel/toolkitEditor/ropEditor/ropEditor.tox differ diff --git a/devel/toolkitEditor/ropEditor/specPanel/specPanel.tox b/devel/toolkitEditor/ropEditor/specPanel/specPanel.tox index 08b44a786..22ea90986 100644 Binary files a/devel/toolkitEditor/ropEditor/specPanel/specPanel.tox and b/devel/toolkitEditor/ropEditor/specPanel/specPanel.tox differ diff --git a/devel/toolkitEditor/testEditor/testEditor.py b/devel/toolkitEditor/testEditor/testEditor.py index b37b8b94d..ffdb0744c 100644 --- a/devel/toolkitEditor/testEditor/testEditor.py +++ b/devel/toolkitEditor/testEditor/testEditor.py @@ -112,12 +112,7 @@ def _create(self, group: str): if not info: print(self.ownerComp, 'there is no rop') return - if group == 'test': - folder = Path(self.ownerComp.par.Testcasefolder.eval()) - elif group == 'snippet': - folder = Path(self.ownerComp.par.Snippetfolder.eval()) - else: - raise Exception(f'Invalid group: {group}') + folder = self._getGroupFolder(group) showPromptDialog( title='Create ' + group, text=group.upper() + ' name', @@ -126,7 +121,41 @@ def _create(self, group: str): ok=lambda name: self._doCreate(name, group, folder), ) - def _doCreate(self, name: str, group: str, folder: Path, confirmed=False): + def _getGroupFolder(self, group: str): + if group == 'test': + return Path(self.ownerComp.par.Testcasefolder.eval()) + elif group == 'snippet': + return Path(self.ownerComp.par.Snippetfolder.eval()) + else: + raise Exception(f'Invalid group: {group}') + + def SaveAs(self): + print(self.ownerComp, 'save as') + info = ext.ropEditor.ROPInfo + if not info: + print(self.ownerComp, 'there is no rop') + return + if not self.currentTox: + print(self.ownerComp, 'there is no tox') + return + currentTox = self.currentTox + if currentTox.endswith('_test.tox'): + group = 'test' + elif currentTox.endswith('_snippet.tox'): + group = 'snippet' + else: + print(self.ownerComp, f'invalid tox name: {currentTox}') + return + folder = self._getGroupFolder(group) + showPromptDialog( + title='Save as', + text='New name', + default=self.currentTestName, + textEntry=True, + ok=lambda name: self._doCreate(name, group, folder, copyExisting=True), + ) + + def _doCreate(self, name: str, group: str, folder: Path, confirmed=False, copyExisting=False): info = ext.ropEditor.ROPInfo if not info: return @@ -143,8 +172,13 @@ def _doCreate(self, name: str, group: str, folder: Path, confirmed=False): text=f'File {toxPath} exists. Replace it?', textEntry=False, ok=lambda _: self._doCreate( - name, group, folder, confirmed=True), + name, group, folder, confirmed=True, copyExisting=copyExisting), ) + elif copyExisting: + comp = iop.loader.Component + if not comp: + raise Exception('No test to copy!') + iop.loader.SaveComponent(toxPath.as_posix()) else: iop.loader.CreateNewComponent( tox=toxPath, diff --git a/devel/toolkitEditor/testEditor/testEditor.tox b/devel/toolkitEditor/testEditor/testEditor.tox index 6c35452a9..b7f488e11 100644 Binary files a/devel/toolkitEditor/testEditor/testEditor.tox and b/devel/toolkitEditor/testEditor/testEditor.tox differ diff --git a/devel/toolkitEditor/toolkitEditor.py b/devel/toolkitEditor/toolkitEditor.py index 49ca7667b..0061f606a 100644 --- a/devel/toolkitEditor/toolkitEditor.py +++ b/devel/toolkitEditor/toolkitEditor.py @@ -23,3 +23,6 @@ def EditROP(self, rop: COMP): @staticmethod def saveAllROPSpecs(): RaytkTools().saveAllROPSpecs() + + def saveAllROPs(self, incrementVersion): + RaytkTools().saveAllROPs(incrementVersion) diff --git a/devel/toolkitEditor/toolkitEditor.tox b/devel/toolkitEditor/toolkitEditor.tox index 84767a2b2..d68fc4940 100644 Binary files a/devel/toolkitEditor/toolkitEditor.tox and b/devel/toolkitEditor/toolkitEditor.tox differ diff --git a/devel/tools/tools.tox b/devel/tools/tools.tox index 035a60f23..873b755da 100644 Binary files a/devel/tools/tools.tox and b/devel/tools/tools.tox differ diff --git a/devel/workArea/workArea.tox b/devel/workArea/workArea.tox index d453fb840..08011e4bf 100644 Binary files a/devel/workArea/workArea.tox and b/devel/workArea/workArea.tox differ diff --git a/docs/_config.yml b/docs/_config.yml index f0418e5b4..e17482a0e 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -29,12 +29,16 @@ github_username: t3kt # Build settings #theme: minima + +# NOTE: Uncomment this for local testing #theme: "just-the-docs" + plugins: - jekyll-redirect-from - jemoji # - jekyll-feed +# NOTE: Comment this out for local testing remote_theme: "pmarsceill/just-the-docs" # Exclude from processing. diff --git a/docs/_data/toolkit.yaml b/docs/_data/toolkit.yaml index de0a5dcde..077d4693e 100644 --- a/docs/_data/toolkit.yaml +++ b/docs/_data/toolkit.yaml @@ -1 +1 @@ -toolkitVersion: '0.39' +toolkitVersion: '0.40' diff --git a/docs/_development/rop-structure.md b/docs/_development/rop-structure.md index e917522d7..26ee3593b 100644 --- a/docs/_development/rop-structure.md +++ b/docs/_development/rop-structure.md @@ -69,7 +69,6 @@ There are several categories of information about a ROP, produced by the `opDefi * `contextType`: the type of context that the ROP's function expects along with the coordinates. * `tags`: indicators that the OP uses certain features like shadows or surface colors. * `definitionPath`: path to the DAT that contains the full table of ROP properties. - * `statePath`: path to the DAT that contains the JSON-serialized `RopState`. * Local definition table fields * These are only included within a single-ROP table inside the `opDefinition`. * `opVersion`: version of that particular type of ROP. @@ -77,11 +76,7 @@ There are several categories of information about a ROP, produced by the `opDefi * `paramSource`: path to the CHOP that holds the values of runtime parameters. * `constantParamSource`: path to the CHOP that holds the values of specialization constant parameters. These are processed separately from runtime parameters to avoid unnecessary cooking. * `paramVectors`: path to a CHOP with the runtime parameters, rearranged into 4 vector channels. This will eventually be used to avoid having to reorder all the parameters in `shaderBuilder`. - * `paramTable`: path to a DAT that contains definitions of all the ROP's parameters. - * `paramTupletTable`: path to a DAT that contains the definitions of the ROP's parameters, organized into 4-part tuplets. * `libraryNames`: names of common shared GLSL libraries and/or paths to ROP-local libraries. - * `elementTable`: path to the DAT that holds information about the ROP's [elements](/raytk/developer/rop-elements/). - * `inputNames`: the names of other ROPs that this ROP's function calls. * `RopState` * This is a structured object stored as JSON in a DAT inside the `opDefinition`. * It holds prepared blocks of code. diff --git a/docs/_layouts/operator.html b/docs/_layouts/operator.html index 8177bc203..0c40e036e 100644 --- a/docs/_layouts/operator.html +++ b/docs/_layouts/operator.html @@ -54,12 +54,21 @@

{{ page.op.name }}

{% if page.op.inputs %}
{% for input in page.op.inputs %} -
- - {{input.label}} - -
-
+ {% if input.sourceParamLabel %} +
+ + ({{input.label}}) + +
+
+ {% else %} +
+ + {{input.label}} + +
+
+ {% endif %} {% endfor %}
{% endif %} @@ -95,7 +104,7 @@

Parameters

{% for param in page.op.parameters %} - {{ param.label | default: param.name }} + {{ param.label | default: param.name }} {{ param.summary | markdownify }} {%include paramHandling.html handling=param.regularHandling %} @@ -135,6 +144,12 @@

Inputs

{% endif %} {{ input.summary }}