Merge pull request #92 from hermfischer-wf/edgar24.1-preview

EDGAR 24.1 Preview

Embedding.py

@@ -662,8 +662,9 @@ def tooManyCells(self,threshold=2):
         group = self.cube.linkroleUri
         cells = int(n/1000000000)
         self.controller.logWarn(f"Presentation group {group} with {axes} axes could have more than {cells} billion cells.  "
-                                +"Split up this presentation group and see EFM 6.25.2 to see how to reduce the number of combinations by selecting "
-                                +"fewer members for each axis."
+                                +"Split up this presentation group and see EXG 9.7.4 to see how to reduce the number of combinations by selecting "
+                                +"fewer members for each axis.",
+                                messageCode="EXG.9.7.4.tooManyCells"
                                 )
         return True
 

Filing.py

@@ -23,11 +23,11 @@
 usGaapOrIfrsPattern = re.compile(".*/fasb[.]org/(us-gaap|srt)/20|.*/xbrl[.]ifrs[.]org/taxonomy/[0-9-]{10}/ifrs-full", re.I)
 deiPattern = re.compile(".*/xbrl[.]sec[.]gov/dei/20", re.I)
 
-def mainFun(controller, modelXbrl, outputFolderName):
+def mainFun(controller, modelXbrl, outputFolderName, transform=None, suplSuffix=None, rFilePrefix=None, altFolder=None, altTransform=None, altSuffix=None):
     if "EdgarRenderer/Filing.py#mainFun" in modelXbrl.arelleUnitTests:
         raise arelle.PythonUtil.pyNamedObject(modelXbrl.arelleUnitTests["EdgarRenderer/Filing.py#mainFun"], "EdgarRenderer/Filing.py#mainFun")
     _funStartedAt = time.time()
-    filing = Filing(controller, modelXbrl, outputFolderName)
+    filing = Filing(controller, modelXbrl, outputFolderName, transform, suplSuffix, rFilePrefix, altFolder, altTransform, altSuffix)
     controller.logDebug("Filing initialized {:.3f} secs.".format(time.time() - _funStartedAt)); _funStartedAt = time.time()
     filing.populateAndLinkClasses()
     controller.logDebug("Filing populateAndLinkClasses {:.3f} secs.".format(time.time() - _funStartedAt)); _funStartedAt = time.time()
@@ -160,8 +160,14 @@ def mainFun(controller, modelXbrl, outputFolderName):
 
 
 class Filing(object):
-    def __init__(self, controller, modelXbrl, outputFolderName):
+    def __init__(self, controller, modelXbrl, outputFolderName, transform, suplSuffix, rFilePrefix, altFolder, altTransform, altSuffix):
         self.modelXbrl = modelXbrl
+        self.transform = transform
+        self.suplSuffix = suplSuffix
+        self.rFilePrefix = rFilePrefix
+        self.altFolder = altFolder
+        self.altTransform = altTransform
+        self.altSuffix = altSuffix
 
         self.cubeDict = {}
         self.axisDict = {}
@@ -209,7 +215,7 @@ def __init__(self, controller, modelXbrl, outputFolderName):
         self.isShr = 'shr' in self.stdNsTokens
 
         self.edgarDocType = next((f.xValue for f in self.modelXbrl.factsByLocalName["DocumentType"]
-                                      if (f.xValue is not None and f.context is not None and not f.context.hasSegment)),None)
+                                      if (f.xValue is not None and f.context is not None and not f.context.hasSegment and f.xValid >= VALID)),None)
 
         self.isFeeExhibit = self.edgarDocType in ['EX-FILING FEES']
 
@@ -283,37 +289,16 @@ def __init__(self, controller, modelXbrl, outputFolderName):
         self.fileNamePrefix = 'R'
         if controller.reportZip:
             self.fileNameBase = None
-            self.dissemFileNameBase = None
             self.reportZip = controller.reportZip
         elif outputFolderName is not None:
             # self.fileNameBase = os.path.normpath(os.path.join(os.path.dirname(controller.webCache.normalizeUrl(modelXbrl.fileSource.basefile)) ,outputFolderName))
             self.fileNameBase = outputFolderName
             if not os.path.exists(self.fileNameBase):  # This is usually the Reports subfolder.
                 os.mkdir(self.fileNameBase)
-            if controller.reportXsltDissem:
-                self.dissemFileNameBase = os.path.join(self.fileNameBase, "dissem")
-                if not os.path.exists(self.dissemFileNameBase):
-                    os.mkdir(self.dissemFileNameBase)
-            else:
-                self.dissemFileNameBase = None
             self.reportZip = None
         else:
             self.fileNameBase = self.reportZip = None
 
-        if controller.reportXslt:
-            _xsltStartedAt = time.time()
-            self.transform = lxml.etree.XSLT(lxml.etree.parse(controller.reportXslt))
-            if controller.reportXsltDissem:
-                self.transformDissem = lxml.etree.XSLT(lxml.etree.parse(controller.reportXsltDissem))
-            else:
-                self.transformDissem = None
-            self.controller.logDebug("Excel XSLT transform {:.3f} secs.".format(time.time() - _xsltStartedAt))
-        ''' HF: this is not used ??
-        if controller.summaryXslt:
-            _xsltStartedAt = time.time()
-            self.summary_transform = lxml.etree.XSLT(lxml.etree.parse(controller.summaryXslt))
-            self.controller.logDebug("Summary XSLT transform {:.3f} secs.".format(time.time() - _xsltStartedAt))
-        '''
         self.reportSummaryList = []
 
         self.rowSeparatorStr = ' | '
@@ -386,7 +371,6 @@ def populateAndLinkClasses(self, uncategorizedCube = None):
 
             # initialize elements
             for qname, factSet in self.modelXbrl.factsByQname.items():
-
                 # we are looking to see if we have "duplicate" facts.  a duplicate fact is one with the same qname, context and unit
                 # as another fact.  Also, keep the first fact with an 'en-US' language, or if there is none, keep the first fact.
                 # the others need to be proactively added to the set of unused facts.
@@ -456,7 +440,7 @@ def factSortKey (fact):
                                                     "Element will be ignored."),
                                                    modelObject=fact, fact=qname)
                         break
-                    elif fact.context is None or (not fact.context.isForeverPeriod and fact.context.endDatetime is None):
+                    elif fact.context is None or fact.xValid < VALID or (not fact.context.isForeverPeriod and fact.context.endDatetime is None):
                         continue # don't break, still might be good facts. we print the error if a context is broken later
 
                     self.elementDict[qname] = Element(fact.concept)
@@ -520,7 +504,11 @@ def factSortKey (fact):
                 self.usedOrBrokenFactDefDict[fact].add(None) #now bad fact won't come back to bite us when processing isUncategorizedFacts
                 continue # fact was rejected in first loop of this function because of problem with the Element
 
-            if fact.unit is None and fact.unitID is not None: # to do a unitref that isn't a unit should be found by arelle, but isn't.
+            if fact.xValid < VALID:
+                self.usedOrBrokenFactDefDict[fact].add(None) #now bad fact won't come back to bite us when processing isUncategorizedFacts
+                continue
+
+            elif fact.unit is None and fact.unitID is not None: # to do a unitref that isn't a unit should be found by arelle, but isn't.
                 if not self.validatedForEFM: # use Arelle validation message
                     self.modelXbrl.error("xbrl.4.6.2:numericUnit",
                          _("Fact %(fact)s context %(contextID)s is numeric and must have a unit"),

Inline.py

@@ -57,6 +57,7 @@ def saveTargetDocumentIfNeeded(cntlr, options, modelXbrl, filing, suffix="_htm."
     filepath, fileext = os.path.splitext(os.path.join(_reportsFolder or "", targetBasename))
     if fileext not in USUAL_INSTANCE_EXTS: fileext = iext
     targetFilename = filepath + fileext
+    modelXbrl.ixTargetFilename = targetFilename
 
     filingZip = None
     filingFiles = None

Report.py

@@ -1151,7 +1151,7 @@ def writeHtmlAndOrXmlFiles(self, reportSummary):
         if self.filing.reportHtmlFormat: self.writeHtmlFile(baseNameBeforeExtension, tree, reportSummary)
 
     def writeXmlFile(self, baseNameBeforeExtension, tree, reportSummary):
-        baseName = baseNameBeforeExtension + '.xml'
+        baseName = (self.filing.rFilePrefix or '') + baseNameBeforeExtension + '.xml' + (self.filing.suplSuffix or '')
         reportSummary.xmlFileName = baseName
         xmlText = treeToString(tree, xml_declaration=True, encoding='utf-8', pretty_print=True)
         if self.filing.reportZip:
@@ -1162,31 +1162,36 @@ def writeXmlFile(self, baseNameBeforeExtension, tree, reportSummary):
             self.controller.renderedFiles.add(baseName)
 
     def writeHtmlFile(self, baseNameBeforeExtension, tree, reportSummary):
-        baseName = baseNameBeforeExtension + '.htm'
+        baseName = (self.filing.rFilePrefix or '') + baseNameBeforeExtension + '.htm' + (self.filing.suplSuffix or '')
         reportSummary.htmlFileName = baseName
-        for _transform, _fileNameBase in (
-            ((self.filing.transform, self.filing.fileNameBase),) + (
-            ((self.filing.transformDissem, self.filing.dissemFileNameBase),) if self.filing.transformDissem else ())):
-            _startedAt = time.time()
-            cell_count = sum(1 for x in tree.iter('Cell'))
-            if cell_count > 50000:
-                self.controller.logWarn(f"There are {cell_count} cells; skipping transformation.")
-                result = fromstring("<HTML><HEAD><TITLE>NOPE</TITLE></HEAD><BODY>NOT TODAY FRIEND</BODY></HTML>")
-            else:
-                keywordArgs= { "asPage" : XSLT.strparam("true") }
-                if getattr(self.embedding, "disclaimer", None) and getattr(self.embedding,"disclaimerStyle",None):
-                    keywordArgs["disclaimer"] = XSLT.strparam(self.embedding.disclaimer)
-                    keywordArgs["disclaimerStyle"] = XSLT.strparam(self.embedding.disclaimerStyle)
-                result = _transform(tree,**keywordArgs)
-            self.controller.logDebug("R{} htm XSLT {:.3f} secs.".format(self.cube.fileNumber, time.time() - _startedAt))
+        _startedAt = time.time()
+        cell_count = sum(1 for x in tree.iter('Cell'))
+        if cell_count > 50000:
+            self.controller.logWarn(f"There are {cell_count} cells; skipping transformation.",
+                                    messageCode="EXG.9.7.renderingCellsLimit")
+            result = fromstring("<HTML><HEAD><TITLE>NOPE</TITLE></HEAD><BODY>Not available</BODY></HTML>")
+        else:
+            keywordArgs= { "asPage" : XSLT.strparam("true") }
+            if getattr(self.embedding, "disclaimer", None) and getattr(self.embedding,"disclaimerStyle",None):
+                keywordArgs["disclaimer"] = XSLT.strparam(self.embedding.disclaimer)
+                keywordArgs["disclaimerStyle"] = XSLT.strparam(self.embedding.disclaimerStyle)
+            result = self.filing.transform(tree,**keywordArgs)
+        htmlText = treeToString(result,method='html',with_tail=False,pretty_print=True,encoding='us-ascii')
+        if self.filing.reportZip:
+            self.filing.reportZip.writestr(baseName, htmlText)
+            self.controller.renderedFiles.add(baseName)
+        elif self.filing.fileNameBase is not None:
+            self.controller.writeFile(os.path.join(self.filing.fileNameBase, baseName), htmlText)
+            self.controller.renderedFiles.add(baseName)
+        if self.filing.altTransform is not None and cell_count <= 50000:
+            # secondary output for workstation
+            baseName = baseNameBeforeExtension + '.htm' + (self.filing.altSuffix or '')
+            reportSummary.htmlFileName = baseName
+            result = self.filing.altTransform(tree,**keywordArgs)
             htmlText = treeToString(result,method='html',with_tail=False,pretty_print=True,encoding='us-ascii')
-            if self.filing.reportZip:
-                self.filing.reportZip.writestr(baseName, htmlText)
-                self.controller.renderedFiles.add(baseName)
-            elif _fileNameBase is not None:
-                self.controller.writeFile(os.path.join(_fileNameBase, baseName), htmlText)
-                if _fileNameBase == self.filing.fileNameBase: # first non-dissem only
-                    self.controller.renderedFiles.add(baseName)
+            self.controller.writeFile(os.path.join(self.filing.altFolder, baseName), htmlText)
+            self.controller.renderedFiles.add(baseName)
+        self.controller.logDebug("R{} htm XSLT {:.3f} secs.".format(self.cube.fileNumber, time.time() - _startedAt))
 
 
     def generateBarChart(self):

Xlout.py

@@ -58,7 +58,7 @@ def close(self):
         del self.simplified_transform
 
 
-    def save(self):
+    def save(self, suffix=""):
         if len(self.wb.worksheets)>1:
             self.wb.remove(self.wb.worksheets[0])
         if not (self.controller.reportZip or self.outputFolderName is not None):
@@ -67,13 +67,14 @@ def save(self):
         file = io.BytesIO()
         self.wb.save(file)
         file.seek(0)
+        outputFileName = OUTPUT_FILE_NAME + suffix
         if self.controller.reportZip:
-            self.controller.reportZip.writestr(OUTPUT_FILE_NAME, file.read())
+            self.controller.reportZip.writestr(outputFileName, file.read())
         else:
-            self.controller.writeFile(os.path.join(self.outputFolderName, OUTPUT_FILE_NAME), file.read())
+            self.controller.writeFile(os.path.join(self.outputFolderName, outputFileName), file.read())
         file.close()
         del file  # dereference
-        self.controller.renderedFiles.add(OUTPUT_FILE_NAME)
+        self.controller.renderedFiles.add(outputFileName)
         self.controller.logDebug('Excel output saved {}'.format(self.controller.entrypoint),file=os.path.basename(__file__))