Merge pull request #96 from hermfischer-wf/edgar24.1-upd1

EDGAR 24.1 Updates

Embedding.py

@@ -662,9 +662,9 @@ def tooManyCells(self,threshold=2):
         group = self.cube.linkroleUri
         cells = int(n/1000000000)
         self.controller.logWarn(f"Presentation group {group} with {axes} axes could have more than {cells} billion cells.  "
-                                +"Split up this presentation group and see EXG 9.7.4 to see how to reduce the number of combinations by selecting "
+                                +"Split up this presentation group and see EXG, Rendering, to see how to reduce the number of combinations by selecting "
                                 +"fewer members for each axis.",
-                                messageCode="EXG.9.7.4.tooManyCells"
+                                messageCode="EXG.rendering.tooManyDimensions"
                                 )
         return True
 

Filing.py

@@ -15,7 +15,7 @@
 from arelle.ModelDtsObject import ModelConcept
 from arelle.ModelObject import ModelObject
 from arelle.XmlUtil import collapseWhitespace
-from arelle.XmlValidate import VALID, VALID_NO_CONTENT
+from arelle.XmlValidateConst import VALID, VALID_NO_CONTENT
 from lxml import etree
 
 from . import Cube, Embedding, Report, PresentationGroup, Summary, Utils, Xlout
@@ -376,6 +376,8 @@ def populateAndLinkClasses(self, uncategorizedCube = None):
                 # the others need to be proactively added to the set of unused facts.
                 if len(factSet) > 1:
                     def factSortKey (fact):
+                        if getattr(fact,"xValid", 0) < VALID:
+                            return ("", "", "")
                         if fact.isNumeric:
                             if fact.isNil: discriminator = float("INF") # Null values always last
                             elif fact.decimals is None: discriminator = 0 # Can happen with invalid xbrl
@@ -389,12 +391,15 @@ def factSortKey (fact):
                     sortedFactList = sorted(factSet, key = factSortKey)
                     while len(sortedFactList) > 0:
                         firstFact = sortedFactList.pop(0)
+                        if getattr(firstFact,"xValid", 0) < VALID:
+                            continue
                         lineNumOfFactWeAreKeeping = firstFact.sourceline
                         discardedLineNumberList = []
                         discardedCounter = 0
                         discardedFactList = []
                         # finds facts with same qname, context and unit as firstFact
                         while (len(sortedFactList) > 0 and
+                               getattr(sortedFactList[0],"xValid", 0) >= VALID and
                                sortedFactList[0].qname == firstFact.qname and
                                sortedFactList[0].context == firstFact.context and
                                sortedFactList[0].unitID == firstFact.unitID):
@@ -419,7 +424,9 @@ def factSortKey (fact):
                                                  linesDiscarded=', '.join(discardedLineNumberList))
 
                 for fact in factSet: # we only want one thing, but we don't want to pop from the set so we "loop" and then break right away
-                    if fact.concept is None:
+                    if getattr(fact,"xValid", 0) < VALID:
+                        continue
+                    elif fact.concept is None:
                         if not self.validatedForEFM:
                             self.modelXbrl.error("xbrl:schemaImportMissing", # use standard Arelle message for this
                                     _("Instance fact missing schema definition: %(elements)s"),
@@ -504,7 +511,7 @@ def factSortKey (fact):
                 self.usedOrBrokenFactDefDict[fact].add(None) #now bad fact won't come back to bite us when processing isUncategorizedFacts
                 continue # fact was rejected in first loop of this function because of problem with the Element
 
-            if fact.xValid < VALID:
+            if getattr(fact,"xValid", 0) < VALID:
                 self.usedOrBrokenFactDefDict[fact].add(None) #now bad fact won't come back to bite us when processing isUncategorizedFacts
                 continue
 

Inline.py

@@ -57,7 +57,6 @@ def saveTargetDocumentIfNeeded(cntlr, options, modelXbrl, filing, suffix="_htm."
     filepath, fileext = os.path.splitext(os.path.join(_reportsFolder or "", targetBasename))
     if fileext not in USUAL_INSTANCE_EXTS: fileext = iext
     targetFilename = filepath + fileext
-    modelXbrl.ixTargetFilename = targetFilename
 
     filingZip = None
     filingFiles = None
@@ -124,6 +123,7 @@ def saveTargetDocument(filing, modelXbrl, targetDocumentFilename, targetDocument
         if getattr(modelXbrl, "isTestcaseVariation", False):
             modelXbrl.extractedInlineInstance = True # for validation comparison
         modelXbrl.modelManager.showStatus(_("Saved extracted instance"), clearAfter=5000)
+        modelXbrl.ixTargetFilename = targetUrl
         return # there can only be one "InlineDocumentSet.CreateTargetInstance" but just to be sure
     cntlr.logTrace(_("Unable to save extracted document, missing plugin class \"InlineDocumentSet.CreateTargetInstance\"."))
 

Report.py

@@ -1168,7 +1168,7 @@ def writeHtmlFile(self, baseNameBeforeExtension, tree, reportSummary):
         cell_count = sum(1 for x in tree.iter('Cell'))
         if cell_count > 50000:
             self.controller.logWarn(f"There are {cell_count} cells; skipping transformation.",
-                                    messageCode="EXG.9.7.renderingCellsLimit")
+                                    messageCode="EXG.rendering.tooManyCells")
             result = fromstring("<HTML><HEAD><TITLE>NOPE</TITLE></HEAD><BODY>Not available</BODY></HTML>")
         else:
             keywordArgs= { "asPage" : XSLT.strparam("true") }

__init__.py

@@ -154,7 +154,8 @@
 from arelle.PluginManager import pluginClassMethods
 from arelle.ValidateFilingText import elementsWithNoContent
 from arelle.XhtmlValidate import xhtmlValidate
-from arelle.XmlValidate import VALID, NONE, validate as xmlValidate
+from arelle.XmlValidateConst import VALID, NONE, UNVALIDATED
+from arelle.XmlValidate import validate as xmlValidate
 from . import RefManager, IoManager, Inline, Utils, Filing, Summary
 import datetime, zipfile, logging, shutil, gettext, time, shlex, sys, traceback, linecache, os, io, tempfile
 import regex as re
@@ -174,7 +175,10 @@ def uncloseSelfClosedTags(doc):
     doc.parser.set_element_class_lookup(None) # modelXbrl class features are already closed now, block class lookup
     for e in doc.xmlRootElement.iter():
         # check if no text, no children and not self-closable element for EDGAR
-        if e.text is None and (not e.getchildren()) and e.tag not in tagsWithNoContent:
+        if (e.text is None and (not e.getchildren())
+            and e.tag not in tagsWithNoContent
+            # also skip ix elements which are nil
+            and not (e.get("{http://www.w3.org/2001/XMLSchema-instance}nil") in ("true","1") and e.tag.startswith("{http://www.xbrl.org/2013/inlineXBRL}"))):
             e.text = "" # prevents self-closing tag with etree.tostring for zip and dissem folders
 
 def allowableBytesForEdgar(bytestr):
@@ -265,7 +269,7 @@ def edgarRendererCmdLineOptionExtender(parser, *args, **kwargs):
     # always use a buffering log handler (even if file or std out)
     parser.add_option("--logToBuffer", action="store_true", dest="logToBuffer", default=True, help=SUPPRESS_HELP)
     parser.add_option("--noRenderingWithError", action="store_true", dest="noRenderingWithError", help=_("Prevent rendering action when exhibit instance validation encountered error(s), blocking R file and extracted xml instance generation for that exhibit instance."))
-
+    parser.add_option("--keepFilingOpen", dest="keepFilingOpen", action="store_true", help=SUPPRESS_HELP) # block closing filing in filingEnd
 
 
 class EdgarRenderer(Cntlr.Cntlr):
@@ -1012,6 +1016,7 @@ def filingEnd(self, cntlr, options, filesource, filing, sourceZipStream=None, *a
         if self.success or not self.noRenderingWithError:
             try:
                 # transform XSLT files
+                reportXslt = None
                 if self.reportXslt:
                     _xsltStartedAt = time.time()
                     reportXslt = etree.XSLT(etree.parse(self.reportXslt))
@@ -1192,6 +1197,7 @@ def copyResourceToReportFolder(filename):
                                 for f in modelXbrl.facts:
                                     if f.get("continuedAt") and hasattr(f, "_ixValue") and f.xValid >= VALID:
                                         del f._ixValue # force rebuilding continuation chain value
+                                        f.xValid = UNVALIDATED
                                         xmlValidate(f.modelXbrl, f, ixFacts=True)
                                 for rel in modelXbrl.relationshipSet("XBRL-footnotes").modelRelationships:
                                     f = rel.toModelObject
@@ -1320,7 +1326,7 @@ def copyResourceToReportFolder(filename):
                                 serializedDoc = fout.read()
                             if not isGUIprivateView:
                                 _filepath.replace("_ht2.xml", "_ht1.xml").replace("_ix2.htm", "_ix1.htm")
-                            filing.writeFile(join(dissemReportsFolder, filename), serializedDoc)
+                            filing.writeFile(os.path.join(dissemReportsFolder, os.path.basename(_filepath)), serializedDoc)
 
 
                     # reissue R files and excel after validation
@@ -1393,8 +1399,8 @@ def copyResourceToReportFolder(filename):
         cntlr.editedModelXbrls.clear()
         cntlr.redactTgtElts.clear()
 
-        # non-GUI (cmd line) options.keepOpen kept modelXbrls open
-        if not cntlr.hasGui and not self.isRunningUnderTestcase():
+        # non-GUI (cmd line) options.keepOpen kept modelXbrls open, use keepFilingOpen to block closing here
+        if not options.keepFilingOpen and not self.isRunningUnderTestcase():
             for report in filing.reports:
                 report.modelXbrl.close()
 
@@ -1675,7 +1681,8 @@ def edgarRendererGuiRun(cntlr, modelXbrl, *args, **kwargs):
             excelXslt = ('InstanceReport_XmlWorkbook.xslt', None)[_combinedReports],
             logMessageTextFile = None,
             logFile = None, # from cntlrCmdLine but need to simulate for GUI operation
-            labelLang = cntlr.labelLang # emulate cmd line labelLang
+            labelLang = cntlr.labelLang, # emulate cmd line labelLang
+            keepFilingOpen = True # closed by CntrlWinMain
         )
         if modelXbrl.modelDocument.type in ModelDocument.Type.TESTCASETYPES:
             modelXbrl.efmOptions = options  # save options in testcase's modelXbrl

resources/arelleMessagesText.xml

@@ -2212,8 +2212,10 @@ Example:
 <message code="dq-{efmSection}-{taxonomy}-Facts-Unexpected">[dq-{efmSection}-{taxonomy}-Facts-Unexpected]Submission type {subType} {severityVerb} not report {taxonomy} namespace facts. {refSources}</message>
 <message code="dq-{efmSection}-{taxonomy}-Version-Required">[dq-{efmSection}-{taxonomy}-Version-Required]Submission type {subType} {severityVerb} use {taxonomy} version {earliestTaxonomy} or later.</message>
 <message code="dq-{efmSection}-Taxonomy-Url-Required">[dq-{efmSection}-Taxonomy-Url-Required]Submission type {subType} document type {docType} {severityVerb} contain a taxonomy URL matching {taxonomyPattern} in its DTS.</message>
+<message code="dq-{efmSection}-Taxonomy-Url-Unexpected">[dq-{efmSection}-Taxonomy-Url-Unexpected]Submission type {subType} document type {docType} {severityVerb} contain a taxonomy URL matching {taxonomyPattern} in its DTS.{msgCoda} {refSources}</message>
 <message code="dq-{efmSection}-{tag}-{value}">[dq-{efmSection}-{tag}-{value}] In submission type {subType}, {tag} value {severityVerb} have {value} in the {context}. {refSources}</message>
 <message code="dq-{efmSection}-{tag}-{expectedValue}">[dq-{efmSection}-{tag}-{value}] In submission type {subType}, {tag} value, {value}, {severityVerb} have {expectedValue} in the {context}. {refSources}</message>
+<message code="dq-{efmSection}-{tag}-ExpectedValue">[dq-{efmSection}-{tag}-ExpectedValue]In submission type {subType}, {tag} value, {value}, {severityVerb} {qualifier}{expectedValue} in the {contextID}. {refSources}</message>
 <message code="dq-{efmSection}-{tag}-Value">[dq-{efmSection}-{tag}-Value] In submission type {subType}, {tag} value, {value}, {severityVerb} be {qualifier}{expectedValue} in the {context}. {refSources}</message>
 <message code="dq-{efmSection}-{tag}-{otherTag}">[dq-{efmSection}-{tag}-{otherTag}] In submission type {subType}, {tag} value {severityVerb} have value {value} in the same context as {otherTag}, {contextID}. {refSources}</message>
 <message code="dq-{efmSection}-{tag}-{otherTag}-Missing">[dq-{efmSection}-{tag}-{otherTag}-Missing] In submission type {subType}, {tag} requires {otherTag} in the same context, {contextID}. {refSources}</message>
@@ -2245,6 +2247,8 @@ Example:
 <message code="dq-0540-CurrentFiscalYearEndDate-Submission-Value">[dq-0540-CurrentFiscalYearEndDate-Submission-Value] In submission type {subType}, {tag} value, {value}, does not match the Submission Header Fiscal Year End Date {valueOfHeaderTag}. {refSources}</message>
 <message code="dq-0540-EntityExTransitionPeriod-EntityEmergingGrowthCompany-Dependency">[dq-0540-EntityExTransitionPeriod-EntityEmergingGrowthCompany-Dependency] In submission type {subType}, EntityExTransitionPeriod must have a value if EntityEmergingGrowthCompany has value &quot;true&quot; in context {contextID}, else it should not be provided. {refSources}</message>
 <message code="dq-0540-{headerTag}-Unexpected">[dq-0540-{headerTag}-Unexpected] Header element {headerTag} value {value} is not applicable for {subType} submission type. {refSources}</message>
+<message code="dq-sro-1yr-period">[dq-sro-1yr-period]In submission type {subType}, {tag} {severityVerb} have a one-year period ending on {otherTag} in context {contextID}. {refSources}</message>
+<message code="dq-sro-monthly-facts">[dq-sro-monthly-facts]In submission type {subType}, there {severityVerb} at least one fact per month of reporting year, missing months {missingMonths}. {refSources}</message>
 
 <!--  DQCRT templated messages -->
 <!-- template