diff --git a/Embedding.py b/Embedding.py index 6cf0350b0..e0306bcf7 100644 --- a/Embedding.py +++ b/Embedding.py @@ -662,8 +662,9 @@ def tooManyCells(self,threshold=2): group = self.cube.linkroleUri cells = int(n/1000000000) self.controller.logWarn(f"Presentation group {group} with {axes} axes could have more than {cells} billion cells. " - +"Split up this presentation group and see EFM 6.25.2 to see how to reduce the number of combinations by selecting " - +"fewer members for each axis." + +"Split up this presentation group and see EXG 9.7.4 to see how to reduce the number of combinations by selecting " + +"fewer members for each axis.", + messageCode="EXG.9.7.4.tooManyCells" ) return True diff --git a/Filing.py b/Filing.py index b3ba9cc48..471b2f013 100644 --- a/Filing.py +++ b/Filing.py @@ -23,11 +23,11 @@ usGaapOrIfrsPattern = re.compile(".*/fasb[.]org/(us-gaap|srt)/20|.*/xbrl[.]ifrs[.]org/taxonomy/[0-9-]{10}/ifrs-full", re.I) deiPattern = re.compile(".*/xbrl[.]sec[.]gov/dei/20", re.I) -def mainFun(controller, modelXbrl, outputFolderName): +def mainFun(controller, modelXbrl, outputFolderName, transform=None, suplSuffix=None, rFilePrefix=None, altFolder=None, altTransform=None, altSuffix=None): if "EdgarRenderer/Filing.py#mainFun" in modelXbrl.arelleUnitTests: raise arelle.PythonUtil.pyNamedObject(modelXbrl.arelleUnitTests["EdgarRenderer/Filing.py#mainFun"], "EdgarRenderer/Filing.py#mainFun") _funStartedAt = time.time() - filing = Filing(controller, modelXbrl, outputFolderName) + filing = Filing(controller, modelXbrl, outputFolderName, transform, suplSuffix, rFilePrefix, altFolder, altTransform, altSuffix) controller.logDebug("Filing initialized {:.3f} secs.".format(time.time() - _funStartedAt)); _funStartedAt = time.time() filing.populateAndLinkClasses() controller.logDebug("Filing populateAndLinkClasses {:.3f} secs.".format(time.time() - _funStartedAt)); _funStartedAt = time.time() @@ -160,8 +160,14 @@ def mainFun(controller, modelXbrl, outputFolderName): class Filing(object): - def __init__(self, controller, modelXbrl, outputFolderName): + def __init__(self, controller, modelXbrl, outputFolderName, transform, suplSuffix, rFilePrefix, altFolder, altTransform, altSuffix): self.modelXbrl = modelXbrl + self.transform = transform + self.suplSuffix = suplSuffix + self.rFilePrefix = rFilePrefix + self.altFolder = altFolder + self.altTransform = altTransform + self.altSuffix = altSuffix self.cubeDict = {} self.axisDict = {} @@ -209,7 +215,7 @@ def __init__(self, controller, modelXbrl, outputFolderName): self.isShr = 'shr' in self.stdNsTokens self.edgarDocType = next((f.xValue for f in self.modelXbrl.factsByLocalName["DocumentType"] - if (f.xValue is not None and f.context is not None and not f.context.hasSegment)),None) + if (f.xValue is not None and f.context is not None and not f.context.hasSegment and f.xValid >= VALID)),None) self.isFeeExhibit = self.edgarDocType in ['EX-FILING FEES'] @@ -283,37 +289,16 @@ def __init__(self, controller, modelXbrl, outputFolderName): self.fileNamePrefix = 'R' if controller.reportZip: self.fileNameBase = None - self.dissemFileNameBase = None self.reportZip = controller.reportZip elif outputFolderName is not None: # self.fileNameBase = os.path.normpath(os.path.join(os.path.dirname(controller.webCache.normalizeUrl(modelXbrl.fileSource.basefile)) ,outputFolderName)) self.fileNameBase = outputFolderName if not os.path.exists(self.fileNameBase): # This is usually the Reports subfolder. os.mkdir(self.fileNameBase) - if controller.reportXsltDissem: - self.dissemFileNameBase = os.path.join(self.fileNameBase, "dissem") - if not os.path.exists(self.dissemFileNameBase): - os.mkdir(self.dissemFileNameBase) - else: - self.dissemFileNameBase = None self.reportZip = None else: self.fileNameBase = self.reportZip = None - if controller.reportXslt: - _xsltStartedAt = time.time() - self.transform = lxml.etree.XSLT(lxml.etree.parse(controller.reportXslt)) - if controller.reportXsltDissem: - self.transformDissem = lxml.etree.XSLT(lxml.etree.parse(controller.reportXsltDissem)) - else: - self.transformDissem = None - self.controller.logDebug("Excel XSLT transform {:.3f} secs.".format(time.time() - _xsltStartedAt)) - ''' HF: this is not used ?? - if controller.summaryXslt: - _xsltStartedAt = time.time() - self.summary_transform = lxml.etree.XSLT(lxml.etree.parse(controller.summaryXslt)) - self.controller.logDebug("Summary XSLT transform {:.3f} secs.".format(time.time() - _xsltStartedAt)) - ''' self.reportSummaryList = [] self.rowSeparatorStr = ' | ' @@ -386,7 +371,6 @@ def populateAndLinkClasses(self, uncategorizedCube = None): # initialize elements for qname, factSet in self.modelXbrl.factsByQname.items(): - # we are looking to see if we have "duplicate" facts. a duplicate fact is one with the same qname, context and unit # as another fact. Also, keep the first fact with an 'en-US' language, or if there is none, keep the first fact. # the others need to be proactively added to the set of unused facts. @@ -456,7 +440,7 @@ def factSortKey (fact): "Element will be ignored."), modelObject=fact, fact=qname) break - elif fact.context is None or (not fact.context.isForeverPeriod and fact.context.endDatetime is None): + elif fact.context is None or fact.xValid < VALID or (not fact.context.isForeverPeriod and fact.context.endDatetime is None): continue # don't break, still might be good facts. we print the error if a context is broken later self.elementDict[qname] = Element(fact.concept) @@ -520,7 +504,11 @@ def factSortKey (fact): self.usedOrBrokenFactDefDict[fact].add(None) #now bad fact won't come back to bite us when processing isUncategorizedFacts continue # fact was rejected in first loop of this function because of problem with the Element - if fact.unit is None and fact.unitID is not None: # to do a unitref that isn't a unit should be found by arelle, but isn't. + if fact.xValid < VALID: + self.usedOrBrokenFactDefDict[fact].add(None) #now bad fact won't come back to bite us when processing isUncategorizedFacts + continue + + elif fact.unit is None and fact.unitID is not None: # to do a unitref that isn't a unit should be found by arelle, but isn't. if not self.validatedForEFM: # use Arelle validation message self.modelXbrl.error("xbrl.4.6.2:numericUnit", _("Fact %(fact)s context %(contextID)s is numeric and must have a unit"), diff --git a/Inline.py b/Inline.py index 8d29292f5..96dd7fadf 100644 --- a/Inline.py +++ b/Inline.py @@ -57,6 +57,7 @@ def saveTargetDocumentIfNeeded(cntlr, options, modelXbrl, filing, suffix="_htm." filepath, fileext = os.path.splitext(os.path.join(_reportsFolder or "", targetBasename)) if fileext not in USUAL_INSTANCE_EXTS: fileext = iext targetFilename = filepath + fileext + modelXbrl.ixTargetFilename = targetFilename filingZip = None filingFiles = None diff --git a/Report.py b/Report.py index bf40f5712..e6b47b948 100644 --- a/Report.py +++ b/Report.py @@ -1151,7 +1151,7 @@ def writeHtmlAndOrXmlFiles(self, reportSummary): if self.filing.reportHtmlFormat: self.writeHtmlFile(baseNameBeforeExtension, tree, reportSummary) def writeXmlFile(self, baseNameBeforeExtension, tree, reportSummary): - baseName = baseNameBeforeExtension + '.xml' + baseName = (self.filing.rFilePrefix or '') + baseNameBeforeExtension + '.xml' + (self.filing.suplSuffix or '') reportSummary.xmlFileName = baseName xmlText = treeToString(tree, xml_declaration=True, encoding='utf-8', pretty_print=True) if self.filing.reportZip: @@ -1162,31 +1162,36 @@ def writeXmlFile(self, baseNameBeforeExtension, tree, reportSummary): self.controller.renderedFiles.add(baseName) def writeHtmlFile(self, baseNameBeforeExtension, tree, reportSummary): - baseName = baseNameBeforeExtension + '.htm' + baseName = (self.filing.rFilePrefix or '') + baseNameBeforeExtension + '.htm' + (self.filing.suplSuffix or '') reportSummary.htmlFileName = baseName - for _transform, _fileNameBase in ( - ((self.filing.transform, self.filing.fileNameBase),) + ( - ((self.filing.transformDissem, self.filing.dissemFileNameBase),) if self.filing.transformDissem else ())): - _startedAt = time.time() - cell_count = sum(1 for x in tree.iter('Cell')) - if cell_count > 50000: - self.controller.logWarn(f"There are {cell_count} cells; skipping transformation.") - result = fromstring("NOPENOT TODAY FRIEND") - else: - keywordArgs= { "asPage" : XSLT.strparam("true") } - if getattr(self.embedding, "disclaimer", None) and getattr(self.embedding,"disclaimerStyle",None): - keywordArgs["disclaimer"] = XSLT.strparam(self.embedding.disclaimer) - keywordArgs["disclaimerStyle"] = XSLT.strparam(self.embedding.disclaimerStyle) - result = _transform(tree,**keywordArgs) - self.controller.logDebug("R{} htm XSLT {:.3f} secs.".format(self.cube.fileNumber, time.time() - _startedAt)) + _startedAt = time.time() + cell_count = sum(1 for x in tree.iter('Cell')) + if cell_count > 50000: + self.controller.logWarn(f"There are {cell_count} cells; skipping transformation.", + messageCode="EXG.9.7.renderingCellsLimit") + result = fromstring("NOPENot available") + else: + keywordArgs= { "asPage" : XSLT.strparam("true") } + if getattr(self.embedding, "disclaimer", None) and getattr(self.embedding,"disclaimerStyle",None): + keywordArgs["disclaimer"] = XSLT.strparam(self.embedding.disclaimer) + keywordArgs["disclaimerStyle"] = XSLT.strparam(self.embedding.disclaimerStyle) + result = self.filing.transform(tree,**keywordArgs) + htmlText = treeToString(result,method='html',with_tail=False,pretty_print=True,encoding='us-ascii') + if self.filing.reportZip: + self.filing.reportZip.writestr(baseName, htmlText) + self.controller.renderedFiles.add(baseName) + elif self.filing.fileNameBase is not None: + self.controller.writeFile(os.path.join(self.filing.fileNameBase, baseName), htmlText) + self.controller.renderedFiles.add(baseName) + if self.filing.altTransform is not None and cell_count <= 50000: + # secondary output for workstation + baseName = baseNameBeforeExtension + '.htm' + (self.filing.altSuffix or '') + reportSummary.htmlFileName = baseName + result = self.filing.altTransform(tree,**keywordArgs) htmlText = treeToString(result,method='html',with_tail=False,pretty_print=True,encoding='us-ascii') - if self.filing.reportZip: - self.filing.reportZip.writestr(baseName, htmlText) - self.controller.renderedFiles.add(baseName) - elif _fileNameBase is not None: - self.controller.writeFile(os.path.join(_fileNameBase, baseName), htmlText) - if _fileNameBase == self.filing.fileNameBase: # first non-dissem only - self.controller.renderedFiles.add(baseName) + self.controller.writeFile(os.path.join(self.filing.altFolder, baseName), htmlText) + self.controller.renderedFiles.add(baseName) + self.controller.logDebug("R{} htm XSLT {:.3f} secs.".format(self.cube.fileNumber, time.time() - _startedAt)) def generateBarChart(self): diff --git a/Xlout.py b/Xlout.py index 53d4f5bc1..12746058e 100644 --- a/Xlout.py +++ b/Xlout.py @@ -58,7 +58,7 @@ def close(self): del self.simplified_transform - def save(self): + def save(self, suffix=""): if len(self.wb.worksheets)>1: self.wb.remove(self.wb.worksheets[0]) if not (self.controller.reportZip or self.outputFolderName is not None): @@ -67,13 +67,14 @@ def save(self): file = io.BytesIO() self.wb.save(file) file.seek(0) + outputFileName = OUTPUT_FILE_NAME + suffix if self.controller.reportZip: - self.controller.reportZip.writestr(OUTPUT_FILE_NAME, file.read()) + self.controller.reportZip.writestr(outputFileName, file.read()) else: - self.controller.writeFile(os.path.join(self.outputFolderName, OUTPUT_FILE_NAME), file.read()) + self.controller.writeFile(os.path.join(self.outputFolderName, outputFileName), file.read()) file.close() del file # dereference - self.controller.renderedFiles.add(OUTPUT_FILE_NAME) + self.controller.renderedFiles.add(outputFileName) self.controller.logDebug('Excel output saved {}'.format(self.controller.entrypoint),file=os.path.basename(__file__)) diff --git a/__init__.py b/__init__.py index b0ceabff6..a984cb396 100644 --- a/__init__.py +++ b/__init__.py @@ -143,16 +143,18 @@ GUI may use tools->language labels setting to override system language for labels """ -VERSION = '3.24.0.1' +VERSION = '3.24.1' from collections import defaultdict from arelle import PythonUtil from arelle import (Cntlr, FileSource, ModelDocument, XmlUtil, Version, ModelValue, Locale, PluginManager, WebCache, ModelFormulaObject, Validate, ViewFileFactList, ViewFileFactTable, ViewFileConcepts, ViewFileFormulae, ViewFileRelationshipSet, ViewFileTests, ViewFileRssFeed, ViewFileRoleTypes) +from arelle.ModelInstanceObject import ModelFact, ModelInlineFootnote from arelle.PluginManager import pluginClassMethods from arelle.ValidateFilingText import elementsWithNoContent -from arelle.XmlValidate import VALID +from arelle.XhtmlValidate import xhtmlValidate +from arelle.XmlValidate import VALID, NONE, validate as xmlValidate from . import RefManager, IoManager, Inline, Utils, Filing, Summary import datetime, zipfile, logging, shutil, gettext, time, shlex, sys, traceback, linecache, os, io, tempfile import regex as re @@ -166,6 +168,7 @@ tagsWithNoContent = set(f"{{http://www.w3.org/1999/xhtml}}{t}" for t in elementsWithNoContent) for t in ("schemaRef", "linkbaseRef", "roleRef", "arcroleRef", "loc", "arc"): tagsWithNoContent.add(f"{{http://www.xbrl.org/2003/linkbase}}{t}") +tagsWithNoContent.add("{http://www.xbrl.org/2013/inlineXBRL}relationship") def uncloseSelfClosedTags(doc): doc.parser.set_element_class_lookup(None) # modelXbrl class features are already closed now, block class lookup @@ -178,6 +181,20 @@ def allowableBytesForEdgar(bytestr): # encode xml-legal ascii bytes not acceptable to EDGAR return re.sub(b"[\\^\x7F]", lambda m: b"&#x%X;" % ord(m[0]), bytestr) +def serializeXml(xmlRootElement): + initialComment = b'' # tostring drops initial comments + node = xmlRootElement + while node.getprevious() is not None: + node = node.getprevious() + if isinstance(node, etree._Comment): + initialComment = etree.tostring(node, encoding="ASCII") + b'\n' + initialComment + serXml = etree.tostring(xmlRootElement, encoding="ASCII", xml_declaration=True) + if initialComment and serXml and serXml.startswith(b" 0): success = False @@ -833,8 +853,14 @@ def processInstance(self, options, modelXbrl, filing, report): if (hasIdAssignedFact or self.isWorkstationFirstPass) and self.reportsFolder: self.cntlr.editedIxDocs[doc.basename] = doc # causes it to be rewritten out self.cntlr.editedModelXbrls.add(modelXbrl) - Inline.saveTargetDocumentIfNeeded(self, options, modelXbrl, filing, - suffix="_ht1." if self.isWorkstationFirstPass else "_htm.") + if self.isWorkstationFirstPass: + if modelXbrl in self.cntlr.editedModelXbrls: + suffix = "_ht2." # private extracted instance for workstation + else: + suffix = "_ht1." # non-private extracted instance for workstation + else: + suffix = "_htm." # extracted instance if not workstation + Inline.saveTargetDocumentIfNeeded(self, options, modelXbrl, filing, suffix=suffix) except Utils.RenderingException as ex: success = False # error message provided at source where exception was raised self.logDebug(_("RenderingException after {} validation errors: {}").format(errorCountDuringValidation, ex)) @@ -846,7 +872,6 @@ def processInstance(self, options, modelXbrl, filing, report): else: self.logWarn(_("The rendering engine was unable to {} due to an internal error. This is not considered an error in the filing.").format(action, errorCountDuringValidation)) self.logDebug(_("Exception traceback: {}").format(traceback.format_exception(*sys.exc_info()))) - del reportSummaryList # dereference self.renderedFiles = filing.renderedFiles # filing-level rendered files if not success: if stripExhibitOnError: @@ -959,6 +984,19 @@ def formatLogMessage(self, logRec): self.logDebug("Message format string error {} in string {}".format(err, logRec.messageCode)) return _text + def transformFilingSummary(self, filing, rootETree, xsltFile, reportsFolder, htmFileName, includeLogs, title=None): + summary_transform = etree.XSLT(etree.parse(xsltFile)) + trargs = {"asPage": etree.XSLT.strparam('true'), + "accessionNumber": "'{}'".format(getattr(filing, "accessionNumber", "")), + "resourcesFolder": "'{}'".format(self.resourcesFolder.replace("\\","/")), + "processXsltInBrowser": etree.XSLT.strparam(str(self.processXsltInBrowser).lower()), + "includeLogs": etree.XSLT.strparam(str(includeLogs).lower()), + "includeExcel": etree.XSLT.strparam("true" if (self.excelXslt) else "false")} + if title: + trargs["title"] = etree.XSLT.strparam(title) + result = summary_transform(rootETree, **trargs) + IoManager.writeHtmlDoc(filing, result, self.reportZip, reportsFolder, htmFileName); + def filingEnd(self, cntlr, options, filesource, filing, sourceZipStream=None, *args, **kwargs): # note that filesource is None if there were no instances if self.abortedDueToMajorError: @@ -967,13 +1005,33 @@ def filingEnd(self, cntlr, options, filesource, filing, sourceZipStream=None, *a # logMessageText needed for successful and unsuccessful termination self.loadLogMessageText() + # GUI operation with redact or redline present requires dissem outputs without new suffixes + hasPrivateData = bool(cntlr.redactTgtElts) or bool(cntlr.redlineIxDocs) + isGUIprivateView = hasPrivateData and cntlr.hasGui + if self.success or not self.noRenderingWithError: try: + # transform XSLT files + if self.reportXslt: + _xsltStartedAt = time.time() + reportXslt = etree.XSLT(etree.parse(self.reportXslt)) + if self.reportXsltDissem: + reportXsltDissem = etree.XSLT(etree.parse(self.reportXsltDissem)) + else: + reportXsltDissem = None + self.logDebug("Excel XSLT transform {:.3f} secs.".format(time.time() - _xsltStartedAt)) + # R files can be produced after knowing if any instance had private data + self.nextFileNum = 1 # important for naming file numbers for multi-instance filings + self.nextUncategorizedFileNum = 9999 + self.nextBarChartFileNum = 0 + rFilePrefix="Private" if hasPrivateData and self.isWorkstationFirstPass else None + for report in filing.reports: + Filing.mainFun(self, report.modelXbrl, self.reportsFolder, transform=reportXslt, rFilePrefix=rFilePrefix) # dissem suffix if self.xlWriter and self.hasXlout: _startedAt = time.time() self.xlWriter.save() self.xlWriter.close() - del self.xlWriter + self.xlWriter = None self.logDebug("Excel saving complete {:.3f} secs.".format(time.time() - _startedAt)) def copyResourceToReportFolder(filename): source = join(self.resourcesFolder, filename) @@ -1011,9 +1069,9 @@ def copyResourceToReportFolder(filename): for filename in set(inputsToCopyToOutputList): # set() to deduplicate if multiple references _filepath = os.path.join(_xbrldir, filename) if filename in cntlr.editedIxDocs: - serializedDoc = allowableBytesForEdgar(etree.tostring(cntlr.editedIxDocs[filename].xmlRootElement, encoding="ASCII", xml_declaration=True)) + serializedDoc = serializeXml(cntlr.editedIxDocs[filename].xmlRootElement) if self.isWorkstationFirstPass: - filename = filename.replace(".htm", "_ix1.htm") + filename = filename.replace(".htm", "_ix2.htm" if hasPrivateData else "_ix1.htm") elif sourceZipStream is not None: with FileSource.openFileSource(_filepath, cntlr, sourceZipStream).file(_filepath, binary=True)[0] as fout: serializedDoc = fout.read() @@ -1057,16 +1115,106 @@ def copyResourceToReportFolder(filename): dissemReportsFolder = None if self.reportZip or self.reportsFolder is not None: IoManager.writeXmlDoc(filing, rootETree, self.reportZip, self.reportsFolder, 'FilingSummary.xml') + # generate supplemental AllReports and other such outputs at this time + for supplReport in pluginClassMethods("EdgarRenderer.FilingEnd.SupplementalReport"): + supplReport(cntlr, filing, self.reportsFolder) # if there's a dissem directory and no logs, remove summary logs - if self.summaryXslt and len(self.summaryXslt) > 0 and (self.summaryXsltDissem or self.reportXsltDissem): + if (hasPrivateData or + self.summaryXslt and len(self.summaryXslt) > 0 and (self.summaryXsltDissem or self.reportXsltDissem)): dissemReportsFolder = os.path.join(self.reportsFolder, "dissem") os.makedirs(dissemReportsFolder, exist_ok=True) if dissemReportsFolder: + redactTgtElts = dict((c.id, c) for c in cntlr.redactTgtElts) # id's assigned during report processing but not necessarily available at load time + removableCntxs = set() + removableUnits = set() + hasRedactedContinuation = False + for f in redactTgtElts.values(): + if isinstance(f, ModelFact): + if f.id in redactTgtElts: + f.xValid = NONE # take out of active model + removableCntxs.add(f.context) + if f.unit is not None: + removableUnits.add(f.unit) + elif f.tag == "{http://www.xbrl.org/2013/inlineXBRL}continuation": + hasRedactedContinuation = True + for report in filing.reports: + modelXbrl = report.modelXbrl + # bypass continuedAt's to redacted elements + if redactTgtElts: # if any redacted continued at elements + for ixdsHtmlRootElt in getattr(modelXbrl, "ixdsHtmlElements", ()): + hasEditedCont = False + for e in ixdsHtmlRootElt.getroottree().iterfind("//{http://www.xbrl.org/2013/inlineXBRL}*[@continuedAt]"): + contAt = e.get("continuedAt", None) + while contAt in redactTgtElts: # may be multiple continuations in redacted sections + nextContAtElt = redactTgtElts[contAt] + if nextContAtElt is not None: + contAt = nextContAtElt.get("continuedAt") + if contAt: + e.set("continuedAt", contAt) + else: + e.attrib.pop("continuedAt", None) + e._continuationElement = getattr(nextContAtElt, "_continuationElement", None) + else: + e.attrib.pop("continuedAt", None) + e._continuationElement = contAt = None + hasEditedCont = True + for e in ixdsHtmlRootElt.iter("{http://www.xbrl.org/2013/inlineXBRL}relationship"): + if any(ref in cntlr.redactTgtElts + for refsAttr in ("fromRefs", "toRefs") + for refs in e.get(refsAttr) + for ref in refs): + hasEditedCont = True + for refsAttr in ("fromRefs", "toRefs"): + refs = (ref for refs in e.get(refsAttr) if ref not in cntlr.redactTgtElts) + if refs: # any refs remain + e.set(' '.join(refs)) + else: + e.getparent().remove(e) # remove this relationship + break + if removableCntxs: # check for orphaned contexts + for e in ixdsHtmlRootElt.iter("{http://www.xbrl.org/2003/instance}context"): + if e in removableCntxs and not any((f.id not in redactTgtElts) for f in modelXbrl.facts if isinstance(f, ModelFact) and f.context == e): + e.getparent().remove(e) # remove this context + e.modelXbrl.contexts.pop(e.id, None) + hasEditedCont = True + if removableUnits: # check for orphaned units + for e in ixdsHtmlRootElt.iter("{http://www.xbrl.org/2003/instance}unit"): + if e in removableUnits and not any((f.id not in redactTgtElts) for f in modelXbrl.facts and f.id not in redactTgtElts and isinstance(f, ModelFact) and f.unit == e): + e.getparent().remove(e) # remove this context + e.modelXbrl.units.pop(e.id, None) + hasEditedCont = True + if hasEditedCont: + doc = ixdsHtmlRootElt.modelDocument + cntlr.redlineIxDocs[doc.basename] = doc # causes it to be rewritten out + cntlr.editedModelXbrls.add(report.modelXbrl) + # rebuild redacted continuation chains + if hasRedactedContinuation: + for f in modelXbrl.facts: + if f.get("continuedAt") and hasattr(f, "_ixValue") and f.xValid >= VALID: + del f._ixValue # force rebuilding continuation chain value + xmlValidate(f.modelXbrl, f, ixFacts=True) + for rel in modelXbrl.relationshipSet("XBRL-footnotes").modelRelationships: + f = rel.toModelObject + if isinstance(f, ModelInlineFootnote): + del f._ixValue # force rebuilding continuation chain value + xmlValidate(f.modelXbrl, f, ixFacts=True) + + # redline-removed docs have self-closed

and other elements which must not be self-closed when saved + # inform user we are schema- and xbrl- revalidating for reportedFile, doc in cntlr.redlineIxDocs.items(): edgarRendererRemoveRedlining(doc) uncloseSelfClosedTags(doc) cntlr.editedIxDocs[reportedFile] = doc # add to editedIxDocs for output in dissem zip and dissem folder + if cntlr.redlineIxDocs: + self.logInfo("Revalidating after redline removal or redaction") + for report in filing.reports: + for ixdsHtmlRootElt in getattr(report.modelXbrl, "ixdsHtmlElements", ()): + if ixdsHtmlRootElt.modelDocument.basename in cntlr.redlineIxDocs: + # revalidate schema validation + xhtmlValidate(report.modelXbrl, ixdsHtmlRootElt) + # revalidate after redaction + Validate.validate(report.modelXbrl) self.renderedFiles.add("FilingSummary.xml") if self.renderingLogsXslt and self.summaryHasLogEntries and not self.processXsltInBrowser: _startedAt = time.time() @@ -1077,32 +1225,19 @@ def copyResourceToReportFolder(filename): self.renderedFiles.add("RenderingLogs.htm") if self.summaryXslt and len(self.summaryXslt) > 0 : _startedAt = time.time() - for _xsltFile, _reportsFolder, _includeLogs in ( - ((self.summaryXslt, self.reportsFolder, True),) + ( - ((self.summaryXsltDissem, dissemReportsFolder, self.includeLogsInSummaryDissem),) - if self.summaryXsltDissem else ())): - if not _xsltFile: continue - summary_transform = etree.XSLT(etree.parse(_xsltFile)) - result = summary_transform(rootETree, asPage=etree.XSLT.strparam('true'), - accessionNumber="'{}'".format(getattr(filing, "accessionNumber", "")), - resourcesFolder="'{}'".format(self.resourcesFolder.replace("\\","/")), - processXsltInBrowser=etree.XSLT.strparam(str(self.processXsltInBrowser).lower()), - includeLogs=etree.XSLT.strparam(str(_includeLogs).lower()), - includeExcel=etree.XSLT.strparam("true" if (self.excelXslt) else "false")) - IoManager.writeHtmlDoc(filing, result, self.reportZip, _reportsFolder, "FilingSummary.htm") - self.logDebug("FilingSummary XSLT transform {:.3f} secs.".format(time.time() - _startedAt)) + self.transformFilingSummary(filing, rootETree, self.summaryXslt, self.reportsFolder, + "PrivateFilingSummary.htm" if hasPrivateData and self.isWorkstationFirstPass else "FilingSummary.htm", + True, + "Private Filing Data" if hasPrivateData else None) self.renderedFiles.add("FilingSummary.htm") + self.logDebug("FilingSummary XSLT transform {:.3f} secs.".format(time.time() - _startedAt)) if self.hasIXBRLViewer: self.renderedFiles.add("ixbrlviewer.html") _startedAt = time.time() for generate in pluginClassMethods("iXBRLViewer.Generate"): generate(cntlr, self.reportsFolder, "/ixviewer-arelle/ixbrlviewer-1.4.11.js", useStubViewer="ixbrlviewer.xhtml", saveStubOnly=True) self.logDebug("Arelle viewer generated {:.3f} secs.".format(time.time() - _startedAt)) - if self.summaryXsltDissem or self.reportXsltDissem: - #print("trace removing summary logs") - summary.removeSummaryLogs() # produce filing summary without logs - IoManager.writeXmlDoc(filing, rootETree, self.reportZip, dissemReportsFolder, 'FilingSummary.xml.dissem') - if self.hasIXBRLViewer: + if self.isWorkstationFirstPass and not hasPrivateData: _startedAt = time.time() for generate in pluginClassMethods("iXBRLViewer.Generate"): generate(cntlr, dissemReportsFolder, "/arelleViewer-1.4.11/ixbrlviewer.js", useStubViewer="ixbrlviewer.xhtml.dissem", saveStubOnly=True) @@ -1133,8 +1268,7 @@ def copyResourceToReportFolder(filename): if reportedFile in cntlr.editedIxDocs: doc = cntlr.editedIxDocs[reportedFile] # redline removed file is not readable in encoded version, create from dom in memory - xbrlZip.writestr(reportedFile, allowableBytesForEdgar( - etree.tostring(doc.xmlRootElement, encoding="ASCII", xml_declaration=True)).decode('utf-8')) + xbrlZip.writestr(reportedFile, serializeXml(doc.xmlRootElement).decode('utf-8')) else: if filesource.isArchive and reportedFile in filesource.dir: _filepath = os.path.join(filesource.baseurl, reportedFile) @@ -1157,16 +1291,78 @@ def copyResourceToReportFolder(filename): # save documents with removed redlines (only when saving dissemReportsFolder) if dissemReportsFolder: + dissemSuffix = ".dissem" if self.isWorkstationFirstPass else "" + inputsToCopyToOutput = set(inputsToCopyToOutputList) - cntlr.editedIxDocs.keys() for reportedFile, modelDocument in cntlr.editedIxDocs.items(): - dissemFilePath = join(dissemReportsFolder, reportedFile) + ".dissem" - filing.writeFile(dissemFilePath, allowableBytesForEdgar( - etree.tostring(modelDocument.xmlRootElement, encoding="ASCII", xml_declaration=True))) - for modelXbrl in cntlr.editedModelXbrls: - if self.isWorkstationFirstPass: # must record in Edgar database in reports folder - Inline.saveTargetDocumentIfNeeded(self, options, modelXbrl, filing) + ix = serializeXml(modelDocument.xmlRootElement) + dissemFilePath = join(dissemReportsFolder, reportedFile) + dissemSuffix + filing.writeFile(dissemFilePath, ix) + if self.isWorkstationFirstPass: # save redacted as .ht1 for workstation + filePath = join(self.reportsFolder, reportedFile).replace(".htm", "_ix1.htm") + filing.writeFile(filePath, ix) + ix = None # dereference + if not self.isWorkstationFirstPass: + shutil.copyfile(os.path.join(self.resourcesFolder, "report.css"), os.path.join(dissemReportsFolder, "report.css")) + for report in filing.reports: + modelXbrl = report.modelXbrl + if modelXbrl in cntlr.editedModelXbrls: + if self.isWorkstationFirstPass: # save redacted as .ht1 for workstation + Inline.saveTargetDocumentIfNeeded(self, options, modelXbrl, filing, suffix="_ht1.") + Inline.saveTargetDocumentIfNeeded(self, options, modelXbrl, filing) # EDGAR dissemination file goes in report folder so it gets into EDGAR database + else: + Inline.saveTargetDocumentIfNeeded(self, options, modelXbrl, filing, altFolder=dissemReportsFolder, suplSuffix=dissemSuffix) + elif hasattr(modelXbrl, "ixTargetFilename"): + inputsToCopyToOutput.add(modelXbrl.ixTargetFilename) + for filename in inputsToCopyToOutput: + if isGUIprivateView or filename.endswith("_ht2.xml") or filename.endswith("_ix2.htm"): + _filepath = os.path.join(self.reportsFolder, filename) + with FileSource.openFileSource(_filepath, cntlr, sourceZipStream).file(_filepath, binary=True)[0] as fout: + serializedDoc = fout.read() + if not isGUIprivateView: + _filepath.replace("_ht2.xml", "_ht1.xml").replace("_ix2.htm", "_ix1.htm") + filing.writeFile(join(dissemReportsFolder, filename), serializedDoc) + + + # reissue R files and excel after validation + if hasPrivateData: + # dissemination and Arelle GUI redacted R file + self.nextFileNum = 1 # important for naming file numbers for multi-instance filings + self.nextUncategorizedFileNum = 9999 + self.nextBarChartFileNum = 0 + self.instanceSummaryList = [] + for report in filing.reports: + if self.isWorkstationFirstPass: + Filing.mainFun(self, report.modelXbrl, dissemReportsFolder, transform=reportXsltDissem, suplSuffix=dissemSuffix, # dissem suffix, Arelle GUI + altFolder=self.reportsFolder, altTransform=reportXslt) # workstation redacted R file + else: # Arelle GUI operation + Filing.mainFun(self, report.modelXbrl, dissemReportsFolder, transform=reportXslt) # no suffix, Arelle GUI + summary = Summary.Summary(self) + rootETree = summary.buildSummaryETree() + summary.removeSummaryLogs() # produce filing summary without logs + if self.isWorkstationFirstPass: # workstation needs redacted filing summary + IoManager.writeXmlDoc(filing, rootETree, self.reportZip, dissemReportsFolder, 'FilingSummary.xml' + dissemSuffix) + if self.summaryXslt: + self.transformFilingSummary(filing, rootETree, self.summaryXslt, self.reportsFolder, "FilingSummary.htm", True, "Public Filing Data") else: - Inline.saveTargetDocumentIfNeeded(self, options, modelXbrl, filing, altFolder=dissemReportsFolder, suplSuffix=".dissem") - + IoManager.writeXmlDoc(filing, rootETree, self.reportZip, dissemReportsFolder, 'FilingSummary.xml' + dissemSuffix) + if self.summaryXslt: + self.transformFilingSummary(filing, rootETree, self.summaryXslt, dissemReportsFolder, "FilingSummary.htm" + dissemSuffix, True, "Public Filing Data") + if self.xlWriter and self.hasXlout: + _startedAt = time.time() + self.xlWriter.save(suffix=dissemSuffix) + self.xlWriter.close() + self.xlWriter = None + self.logDebug("Excel saving complete {:.3f} secs.".format(time.time() - _startedAt)) + # generate supplemental AllReports and other such outputs at this time + for supplReport in pluginClassMethods("EdgarRenderer.FilingEnd.SupplementalReport"): + supplReport(cntlr, filing, dissemReportsFolder) + if self.hasIXBRLViewer: + _startedAt = time.time() + for generate in pluginClassMethods("iXBRLViewer.Generate"): + generate(cntlr, dissemReportsFolder, "/arelleViewer-1.4.11/ixbrlviewer.js", + useStubViewer="ixbrlviewer.xhtml.dissem" if self.isWorkstationFirstPass else "ixbrlviewer.xhtml", + saveStubOnly=True) + self.logDebug("Arelle viewer for dissemination generated {:.3f} secs.".format(time.time() - _startedAt)) if "EdgarRenderer/__init__.py#filingEnd" in filing.arelleUnitTests: raise arelle.PythonUtil.pyNamedObject(filing.arelleUnitTests["EdgarRenderer/__init__.py#filingEnd"], "EdgarRenderer/__init__.py#filingEnd") @@ -1195,6 +1391,12 @@ def copyResourceToReportFolder(filename): cntlr.editedIxDocs.clear() # deref modelXbrls even if unsuccessful cntlr.redlineIxDocs.clear() cntlr.editedModelXbrls.clear() + cntlr.redactTgtElts.clear() + + # non-GUI (cmd line) options.keepOpen kept modelXbrls open + if not cntlr.hasGui and not self.isRunningUnderTestcase(): + for report in filing.reports: + report.modelXbrl.close() # close filesource (which may have been an archive), regardless of success above filesource.close() @@ -1321,31 +1523,19 @@ def postprocessFailure(self, options): def addToLog(self, message, messageArgs={}, messageCode='error', file=MODULENAME, level=logging.DEBUG): - # Master log and error/warning msg handler - messageDict = {'fatal':logging.FATAL - , 'error':logging.ERROR - , 'warn':logging.WARN - , 'info':logging.INFO - , 'debug':logging.DEBUG - , 'trace':logging.NOTSET} - # find a level that agrees with the code - if messageCode not in messageDict: messageLevel = logging.CRITICAL - else: messageLevel = messageDict[messageCode.casefold()] - # if both level and code were given, err on the side of more logging: - messageLevel = max(level, messageLevel) if self.entrypoint is not None and len(self.instanceList + self.inlineList) > 1: message += ' --' + (self.entrypoint.url if isinstance(self.entrypoint,FileSource.FileSource) else self.entrypoint) message = message.encode('utf-8', 'replace').decode('utf-8') - if messageLevel >= logging.INFO: + if level >= logging.INFO: self.ErrorMsgs.append(Utils.Errmsg(messageCode, message)) # dereference non-string messageArg values messageArgs = dict((k,str(v)) for k,v in messageArgs.items()) if (self.modelManager and getattr(self.modelManager, 'modelXbrl', None)): - self.modelManager.modelXbrl.log(logging.getLevelName(messageLevel), messageCode, message, *messageArgs) + self.modelManager.modelXbrl.log(logging.getLevelName(level), messageCode, message, *messageArgs) else: - self.cntlr.addToLog(message, messageArgs=messageArgs, messageCode=messageCode, file=file, level=messageLevel) + self.cntlr.addToLog(message, messageArgs=messageArgs, messageCode=messageCode, file=file, level=level) # Lowercase tokens apparently write to standard output?? @@ -1391,7 +1581,7 @@ def edgarRendererGuiViewMenuExtender(cntlr, viewMenu, *args, **kwargs): def setShowFilingData(self, *args): cntlr.config["edgarRendererShowFilingData"] = cntlr.showFilingData.get() cntlr.saveConfig() - erViewMenu.entryconfig("Show Redlining", state="normal" if cntlr.showFilingData.get() else "disabled") + erViewMenu.entryconfig("Show Redlining and Redactions", state="normal" if cntlr.showFilingData.get() else "disabled") def setRedlineMode(self, *args): cntlr.config["edgarRendererRedlineMode"] = cntlr.redlineMode.get() cntlr.saveConfig() @@ -1406,7 +1596,7 @@ def setValidateBeforeRendering(self, *args): erViewMenu.add_checkbutton(label=_("Show Filing Data"), underline=0, variable=cntlr.showFilingData, onvalue=True, offvalue=False) cntlr.redlineMode = BooleanVar(value=cntlr.config.get("edgarRendererRedlineMode", True)) cntlr.redlineMode.trace("w", setRedlineMode) - erViewMenu.add_checkbutton(label=_("Show Redlining"), underline=0, variable=cntlr.redlineMode, onvalue=True, offvalue=False, + erViewMenu.add_checkbutton(label=_("Show Redlining and Redactions"), underline=0, variable=cntlr.redlineMode, onvalue=True, offvalue=False, state="normal" if cntlr.showFilingData.get() else "disabled") cntlr.showTablesMenu = BooleanVar(value=cntlr.config.get("edgarRendererShowTablesMenu", True)) cntlr.showTablesMenu.trace("w", setShowTablesMenu) @@ -1430,21 +1620,16 @@ def edgarRendererGuiRun(cntlr, modelXbrl, *args, **kwargs): if "summaryXslt" in parameters and "reportXslt" in parameters: _reportXslt = parameters["reportXslt"][1] _summaryXslt = parameters["summaryXslt"][1] - if "ixRedline" in parameters and parameters["ixRedline"][1] == "true": - _ixRedline = "?redline=true" - else: - _ixRedline = "" + _ixRedline = "ixRedline" in parameters and parameters["ixRedline"][1] == "true" else: _reportXslt = ('InstanceReport.xslt', 'InstanceReportTable.xslt')[_combinedReports] _summaryXslt = ('Summarize.xslt', '')[_combinedReports] # no FilingSummary.htm for Rall.htm production - if cntlr.redlineMode.get(): - _ixRedline = "?redline=true" - else: - _ixRedline = "" + _ixRedline = cntlr.redlineMode.get() if not hasattr(cntlr, "editedIxDocs"): cntlr.editedIxDocs = {} cntlr.editedModelXbrls = set() cntlr.redlineIxDocs = {} + cntlr.redactTgtElts = set() isNonEFMorGFMinline = (not getattr(cntlr.modelManager.disclosureSystem, "EFMplugin", False) and modelXbrl.modelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET)) # may use GUI mode to process a single instance or test suite @@ -1569,6 +1754,7 @@ def addRefDocs(doc): if instanceModelDocument.type == ModelDocument.Type.INLINEXBRLDOCUMENTSET: uri = instanceModelDocument.targetDocumentPreferredFilename.replace(".xbrl",".htm") report = PythonUtil.attrdict( # simulate report + modelXbrl = instanceModelXbrl, isInline = instanceModelDocument.type in (ModelDocument.Type.INLINEXBRL, ModelDocument.Type.INLINEXBRLDOCUMENTSET), reportedFiles = reportedFiles, renderedFiles = set(), @@ -1587,6 +1773,8 @@ def addRefDocs(doc): edgarRendererXbrlRun(cntlr, options, instanceModelXbrl, filing, report) edgarRenderer = filing.edgarRenderer reportsFolder = edgarRenderer.reportsFolder + hasRedactOrRedlineElts = bool(cntlr.redactTgtElts) or bool(cntlr.redlineIxDocs) + edgarRendererFilingEnd(cntlr, options, modelXbrl.fileSource, filing) cntlr.logHandler.endLogBuffering() # block other GUI processes from using log buffer ''' @@ -1632,17 +1820,25 @@ def addRefDocs(doc): if cntlr.showFilingData.get(): from . import LocalViewer _localhost = LocalViewer.init(cntlr, reportsFolder) + if hasRedactOrRedlineElts: + _localhostDissem = LocalViewer.init(cntlr, reportsFolder + "/dissem") import webbrowser - openingUrl = None + openingUrl = openingUrlDissem = None if isNonEFMorGFMinline: # for non-EFM/GFM open ix viewer directly filingSummaryTree = etree.parse(os.path.join(edgarRenderer.reportsFolder, "FilingSummary.xml")) for reportElt in filingSummaryTree.iter(tag="Report"): if reportElt.get("instance"): openingUrl = f"ix?doc=/{_localhost.rpartition('/')[2]}/{reportElt.get('instance')}&xbrl=true" - break + if hasRedactOrRedlineElts: + openingUrlDissem = f"ix?doc=/{_localhostDissem.rpartition('/')[2]}{reportElt.get('instance')}&xbrl=true" if not openingUrl: # open SEC Mustard Menu openingUrl = ("FilingSummary.htm", "Rall.htm")[_combinedReports] - webbrowser.open(url="{}/{}{}".format(_localhost, openingUrl, _ixRedline)) + if hasRedactOrRedlineElts: + openingUrlDissem = ("FilingSummary.htm", "Rall.htm")[_combinedReports] + webbrowser.open(url="{}/{}{}".format(_localhost, openingUrl, + "?redline=true" if (_ixRedline and hasRedactOrRedlineElts) else "")) + if hasRedactOrRedlineElts: + webbrowser.open(url="{}/{}".format(_localhostDissem, openingUrlDissem)) if filing.edgarRenderer.hasIXBRLViewer and filing.hasInlineReport: webbrowser.open(url="{}/ixbrlviewer.xhtml{}".format(_localhost, _ixRedline)) @@ -1656,43 +1852,56 @@ def testcaseVariationExpectedSeverity(modelTestcaseVariation, *args, **kwargs): def savesTargetInstance(*args, **kwargs): # EdgarRenderer implements its own target instance saver return True -redliningPattern = re.compile(r"(.*;)?\s*-sec-ix-redline\s*:\s*true(?:\s*;)?\s*([\w.-].*)?$") +redliningPattern = re.compile(r"(.*;)?\s*-sec-ix-(redline|redact)\s*:\s*true(?:\s*;)?\s*([\w.-].*)?$") def edgarRendererDetectRedlining(modelDocument, *args, **kwargs): cntlr = modelDocument.modelXbrl.modelManager.cntlr - if modelDocument.type == ModelDocument.Type.INLINEXBRL and (not cntlr.hasGui or not cntlr.redlineMode.get()): + foundMatchInDoc = False + if modelDocument.type == ModelDocument.Type.INLINEXBRL and (not cntlr.hasGui or cntlr.redlineMode.get()): for e in modelDocument.xmlRootElement.getroottree().iterfind("//{http://www.w3.org/1999/xhtml}*[@style]"): - if redliningPattern.match(e.get("style","")): + rlMatch = redliningPattern.match(e.get("style","")) + if rlMatch: if not hasattr(cntlr, "editedIxDocs"): cntlr.editedIxDocs = {} cntlr.editedModelXbrls = set() cntlr.redlineIxDocs = {} - cntlr.redlineIxDocs[modelDocument.basename] = modelDocument - break + cntlr.redactTgtElts = set() + if not foundMatchInDoc: + cntlr.redlineIxDocs[modelDocument.basename] = modelDocument + foundMatchInDoc = True + if rlMatch.group(2) == "redact": + for c in e.iter("{http://www.xbrl.org/2013/inlineXBRL}*"): + cntlr.redactTgtElts.add(c) def edgarRendererRemoveRedlining(modelDocument, *args, **kwargs): # strip redlining from modelDocument + matchedElts = [] for e in modelDocument.xmlRootElement.getroottree().iterfind("//{http://www.w3.org/1999/xhtml}*[@style]"): rlMatch = redliningPattern.match(e.get("style","")) if rlMatch: - rlRemoved = True - cleanedStyle = (rlMatch.group(1) or "") + (rlMatch.group(2) or "") - if cleanedStyle: + matchedElts.append(e) # can't prune tree while iterating through it + for e in matchedElts: + rlMatch = redliningPattern.match(e.get("style","")) + if rlMatch: + isRedact = rlMatch.group(2) == "redact" + cleanedStyle = (rlMatch.group(1) or "") + (rlMatch.group(3) or "") + if cleanedStyle and not isRedact: e.set("style", cleanedStyle) else: e.attrib.pop("style") # if no remaining attributes on remove it - if not e.attrib and e.tag == "{http://www.w3.org/1999/xhtml}span": + if isRedact or (not e.attrib and e.tag == "{http://www.w3.org/1999/xhtml}span"): e0 = e.getprevious() prop = "tail" if e0 is None: e0 = e.getparent() prop = "text" - if e.text: - setattr(e0, prop, (getattr(e0, prop) or "") + e.text) - for eChild in e.getchildren(): - e.addprevious(eChild) - e0 = eChild - prop = "tail" + if not isRedact: # redline - move children to parent + if e.text: + setattr(e0, prop, (getattr(e0, prop) or "") + e.text) + for eChild in e.getchildren(): + e.addprevious(eChild) + e0 = eChild + prop = "tail" if e.tail: setattr(e0, prop, (getattr(e0, prop) or "") + e.tail) e.getparent().remove(e) diff --git a/ix b/ix index 8147fad40..88592a950 100644 --- a/ix +++ b/ix @@ -5,8 +5,8 @@ - - EDGAR Inline XBRL Viewer + + XBRL Viewer - + +