diff --git a/Embedding.py b/Embedding.py index e0306bcf..4a5e7bdb 100644 --- a/Embedding.py +++ b/Embedding.py @@ -662,9 +662,9 @@ def tooManyCells(self,threshold=2): group = self.cube.linkroleUri cells = int(n/1000000000) self.controller.logWarn(f"Presentation group {group} with {axes} axes could have more than {cells} billion cells. " - +"Split up this presentation group and see EXG 9.7.4 to see how to reduce the number of combinations by selecting " + +"Split up this presentation group and see EXG, Rendering, to see how to reduce the number of combinations by selecting " +"fewer members for each axis.", - messageCode="EXG.9.7.4.tooManyCells" + messageCode="EXG.rendering.tooManyDimensions" ) return True diff --git a/Filing.py b/Filing.py index 471b2f01..20f7a69b 100644 --- a/Filing.py +++ b/Filing.py @@ -15,7 +15,7 @@ from arelle.ModelDtsObject import ModelConcept from arelle.ModelObject import ModelObject from arelle.XmlUtil import collapseWhitespace -from arelle.XmlValidate import VALID, VALID_NO_CONTENT +from arelle.XmlValidateConst import VALID, VALID_NO_CONTENT from lxml import etree from . import Cube, Embedding, Report, PresentationGroup, Summary, Utils, Xlout @@ -376,6 +376,8 @@ def populateAndLinkClasses(self, uncategorizedCube = None): # the others need to be proactively added to the set of unused facts. if len(factSet) > 1: def factSortKey (fact): + if getattr(fact,"xValid", 0) < VALID: + return ("", "", "") if fact.isNumeric: if fact.isNil: discriminator = float("INF") # Null values always last elif fact.decimals is None: discriminator = 0 # Can happen with invalid xbrl @@ -389,12 +391,15 @@ def factSortKey (fact): sortedFactList = sorted(factSet, key = factSortKey) while len(sortedFactList) > 0: firstFact = sortedFactList.pop(0) + if getattr(firstFact,"xValid", 0) < VALID: + continue lineNumOfFactWeAreKeeping = firstFact.sourceline discardedLineNumberList = [] discardedCounter = 0 discardedFactList = [] # finds facts with same qname, context and unit as firstFact while (len(sortedFactList) > 0 and + getattr(sortedFactList[0],"xValid", 0) >= VALID and sortedFactList[0].qname == firstFact.qname and sortedFactList[0].context == firstFact.context and sortedFactList[0].unitID == firstFact.unitID): @@ -419,7 +424,9 @@ def factSortKey (fact): linesDiscarded=', '.join(discardedLineNumberList)) for fact in factSet: # we only want one thing, but we don't want to pop from the set so we "loop" and then break right away - if fact.concept is None: + if getattr(fact,"xValid", 0) < VALID: + continue + elif fact.concept is None: if not self.validatedForEFM: self.modelXbrl.error("xbrl:schemaImportMissing", # use standard Arelle message for this _("Instance fact missing schema definition: %(elements)s"), @@ -504,7 +511,7 @@ def factSortKey (fact): self.usedOrBrokenFactDefDict[fact].add(None) #now bad fact won't come back to bite us when processing isUncategorizedFacts continue # fact was rejected in first loop of this function because of problem with the Element - if fact.xValid < VALID: + if getattr(fact,"xValid", 0) < VALID: self.usedOrBrokenFactDefDict[fact].add(None) #now bad fact won't come back to bite us when processing isUncategorizedFacts continue diff --git a/Inline.py b/Inline.py index 96dd7fad..8ea098dd 100644 --- a/Inline.py +++ b/Inline.py @@ -57,7 +57,6 @@ def saveTargetDocumentIfNeeded(cntlr, options, modelXbrl, filing, suffix="_htm." filepath, fileext = os.path.splitext(os.path.join(_reportsFolder or "", targetBasename)) if fileext not in USUAL_INSTANCE_EXTS: fileext = iext targetFilename = filepath + fileext - modelXbrl.ixTargetFilename = targetFilename filingZip = None filingFiles = None @@ -124,6 +123,7 @@ def saveTargetDocument(filing, modelXbrl, targetDocumentFilename, targetDocument if getattr(modelXbrl, "isTestcaseVariation", False): modelXbrl.extractedInlineInstance = True # for validation comparison modelXbrl.modelManager.showStatus(_("Saved extracted instance"), clearAfter=5000) + modelXbrl.ixTargetFilename = targetUrl return # there can only be one "InlineDocumentSet.CreateTargetInstance" but just to be sure cntlr.logTrace(_("Unable to save extracted document, missing plugin class \"InlineDocumentSet.CreateTargetInstance\".")) diff --git a/Report.py b/Report.py index e6b47b94..d01b5fa3 100644 --- a/Report.py +++ b/Report.py @@ -1168,7 +1168,7 @@ def writeHtmlFile(self, baseNameBeforeExtension, tree, reportSummary): cell_count = sum(1 for x in tree.iter('Cell')) if cell_count > 50000: self.controller.logWarn(f"There are {cell_count} cells; skipping transformation.", - messageCode="EXG.9.7.renderingCellsLimit") + messageCode="EXG.rendering.tooManyCells") result = fromstring("NOPENot available") else: keywordArgs= { "asPage" : XSLT.strparam("true") } diff --git a/__init__.py b/__init__.py index a984cb39..825339a0 100644 --- a/__init__.py +++ b/__init__.py @@ -154,7 +154,8 @@ from arelle.PluginManager import pluginClassMethods from arelle.ValidateFilingText import elementsWithNoContent from arelle.XhtmlValidate import xhtmlValidate -from arelle.XmlValidate import VALID, NONE, validate as xmlValidate +from arelle.XmlValidateConst import VALID, NONE, UNVALIDATED +from arelle.XmlValidate import validate as xmlValidate from . import RefManager, IoManager, Inline, Utils, Filing, Summary import datetime, zipfile, logging, shutil, gettext, time, shlex, sys, traceback, linecache, os, io, tempfile import regex as re @@ -174,7 +175,10 @@ def uncloseSelfClosedTags(doc): doc.parser.set_element_class_lookup(None) # modelXbrl class features are already closed now, block class lookup for e in doc.xmlRootElement.iter(): # check if no text, no children and not self-closable element for EDGAR - if e.text is None and (not e.getchildren()) and e.tag not in tagsWithNoContent: + if (e.text is None and (not e.getchildren()) + and e.tag not in tagsWithNoContent + # also skip ix elements which are nil + and not (e.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true","1") and e.tag.startswith("{http://www.xbrl.org/2013/inlineXBRL}"))): e.text = "" # prevents self-closing tag with etree.tostring for zip and dissem folders def allowableBytesForEdgar(bytestr): @@ -265,7 +269,7 @@ def edgarRendererCmdLineOptionExtender(parser, *args, **kwargs): # always use a buffering log handler (even if file or std out) parser.add_option("--logToBuffer", action="store_true", dest="logToBuffer", default=True, help=SUPPRESS_HELP) parser.add_option("--noRenderingWithError", action="store_true", dest="noRenderingWithError", help=_("Prevent rendering action when exhibit instance validation encountered error(s), blocking R file and extracted xml instance generation for that exhibit instance.")) - + parser.add_option("--keepFilingOpen", dest="keepFilingOpen", action="store_true", help=SUPPRESS_HELP) # block closing filing in filingEnd class EdgarRenderer(Cntlr.Cntlr): @@ -1012,6 +1016,7 @@ def filingEnd(self, cntlr, options, filesource, filing, sourceZipStream=None, *a if self.success or not self.noRenderingWithError: try: # transform XSLT files + reportXslt = None if self.reportXslt: _xsltStartedAt = time.time() reportXslt = etree.XSLT(etree.parse(self.reportXslt)) @@ -1192,6 +1197,7 @@ def copyResourceToReportFolder(filename): for f in modelXbrl.facts: if f.get("continuedAt") and hasattr(f, "_ixValue") and f.xValid >= VALID: del f._ixValue # force rebuilding continuation chain value + f.xValid = UNVALIDATED xmlValidate(f.modelXbrl, f, ixFacts=True) for rel in modelXbrl.relationshipSet("XBRL-footnotes").modelRelationships: f = rel.toModelObject @@ -1320,7 +1326,7 @@ def copyResourceToReportFolder(filename): serializedDoc = fout.read() if not isGUIprivateView: _filepath.replace("_ht2.xml", "_ht1.xml").replace("_ix2.htm", "_ix1.htm") - filing.writeFile(join(dissemReportsFolder, filename), serializedDoc) + filing.writeFile(os.path.join(dissemReportsFolder, os.path.basename(_filepath)), serializedDoc) # reissue R files and excel after validation @@ -1393,8 +1399,8 @@ def copyResourceToReportFolder(filename): cntlr.editedModelXbrls.clear() cntlr.redactTgtElts.clear() - # non-GUI (cmd line) options.keepOpen kept modelXbrls open - if not cntlr.hasGui and not self.isRunningUnderTestcase(): + # non-GUI (cmd line) options.keepOpen kept modelXbrls open, use keepFilingOpen to block closing here + if not options.keepFilingOpen and not self.isRunningUnderTestcase(): for report in filing.reports: report.modelXbrl.close() @@ -1675,7 +1681,8 @@ def edgarRendererGuiRun(cntlr, modelXbrl, *args, **kwargs): excelXslt = ('InstanceReport_XmlWorkbook.xslt', None)[_combinedReports], logMessageTextFile = None, logFile = None, # from cntlrCmdLine but need to simulate for GUI operation - labelLang = cntlr.labelLang # emulate cmd line labelLang + labelLang = cntlr.labelLang, # emulate cmd line labelLang + keepFilingOpen = True # closed by CntrlWinMain ) if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES: modelXbrl.efmOptions = options # save options in testcase's modelXbrl diff --git a/resources/arelleMessagesText.xml b/resources/arelleMessagesText.xml index 90bf3799..11564738 100644 --- a/resources/arelleMessagesText.xml +++ b/resources/arelleMessagesText.xml @@ -2212,8 +2212,10 @@ Example: [dq-{efmSection}-{taxonomy}-Facts-Unexpected]Submission type {subType} {severityVerb} not report {taxonomy} namespace facts. {refSources} [dq-{efmSection}-{taxonomy}-Version-Required]Submission type {subType} {severityVerb} use {taxonomy} version {earliestTaxonomy} or later. [dq-{efmSection}-Taxonomy-Url-Required]Submission type {subType} document type {docType} {severityVerb} contain a taxonomy URL matching {taxonomyPattern} in its DTS. +[dq-{efmSection}-Taxonomy-Url-Unexpected]Submission type {subType} document type {docType} {severityVerb} contain a taxonomy URL matching {taxonomyPattern} in its DTS.{msgCoda} {refSources} [dq-{efmSection}-{tag}-{value}] In submission type {subType}, {tag} value {severityVerb} have {value} in the {context}. {refSources} [dq-{efmSection}-{tag}-{value}] In submission type {subType}, {tag} value, {value}, {severityVerb} have {expectedValue} in the {context}. {refSources} +[dq-{efmSection}-{tag}-ExpectedValue]In submission type {subType}, {tag} value, {value}, {severityVerb} {qualifier}{expectedValue} in the {contextID}. {refSources} [dq-{efmSection}-{tag}-Value] In submission type {subType}, {tag} value, {value}, {severityVerb} be {qualifier}{expectedValue} in the {context}. {refSources} [dq-{efmSection}-{tag}-{otherTag}] In submission type {subType}, {tag} value {severityVerb} have value {value} in the same context as {otherTag}, {contextID}. {refSources} [dq-{efmSection}-{tag}-{otherTag}-Missing] In submission type {subType}, {tag} requires {otherTag} in the same context, {contextID}. {refSources} @@ -2245,6 +2247,8 @@ Example: [dq-0540-CurrentFiscalYearEndDate-Submission-Value] In submission type {subType}, {tag} value, {value}, does not match the Submission Header Fiscal Year End Date {valueOfHeaderTag}. {refSources} [dq-0540-EntityExTransitionPeriod-EntityEmergingGrowthCompany-Dependency] In submission type {subType}, EntityExTransitionPeriod must have a value if EntityEmergingGrowthCompany has value "true" in context {contextID}, else it should not be provided. {refSources} [dq-0540-{headerTag}-Unexpected] Header element {headerTag} value {value} is not applicable for {subType} submission type. {refSources} +[dq-sro-1yr-period]In submission type {subType}, {tag} {severityVerb} have a one-year period ending on {otherTag} in context {contextID}. {refSources} +[dq-sro-monthly-facts]In submission type {subType}, there {severityVerb} at least one fact per month of reporting year, missing months {missingMonths}. {refSources}