OPEN: FP integration (v2)

Deeploy/AbstractDataTypes.py

@@ -26,6 +26,7 @@
 from __future__ import annotations
 
 import copy
+import math
 from abc import abstractmethod
 from dataclasses import dataclass
 from typing import Dict, Generic, Iterable, List, Optional, Type, TypeVar, Union
@@ -234,6 +235,74 @@ def checkValue(cls, value: Union[int, Iterable[int]], ctxt: Optional[_NetworkCon
         return True
 
 
+class FloatImmediate(Immediate[Union[float, Iterable[float]], _ImmediateType]):
+    typeMantissa: int  #: int: Represents the number of bits reserved for the mantissa part
+    typeExponent: int  #: int: Represents the number of bits reserved for the exponent part
+
+    @_classproperty
+    def typeExponentMax(cls) -> int:
+        # In floating point, all 1 in exponent is reserved for special numbers (i.e. NaN or Inf)
+        return 2**(cls.typeExponent) - 2
+
+    @_classproperty
+    def typeExponentOffset(cls) -> int:
+        # The offset added to the exponent
+        return 2**(cls.typeExponent - 1) - 1
+
+    @classmethod
+    def partialOrderUpcast(cls, otherCls: Type[Immediate]) -> bool:
+        if issubclass(otherCls, FloatImmediate):
+            return cls.typeMantissa >= otherCls.typeMantissa and cls.typeExponent >= otherCls.typeExponent
+        else:
+            return False
+
+    @classmethod
+    def checkValue(cls, value: Union[float, Iterable[float]], ctxt: Optional[_NetworkContext] = None):
+        """
+        This method tries to manually cast standard python's standard immediate float precision values 
+        (64 bits) to an arbitrary FP representation and check if the new representation is close enough 
+        to the original value.
+        """
+        _val_list = []
+
+        if isinstance(value, float):
+            _val_list.append(value)
+        elif isinstance(value, np.ndarray):
+            _val_list = value.tolist()
+        elif isinstance(value, Iterable):
+            for i in value:
+                _val_list.append(i)
+        else:
+            raise Exception("Immediate type not recognized.")
+
+        # The exponent bias for FP64 is 2**(11-1)-1 as the exponent has 11 bits.
+        DOUBLE_MIN_EXP = -1023
+
+        for val in _val_list:
+
+            # Extract mantissa, exponent, and sign.
+            # Also bring mantissa and exponent to IEEE754 compliant form for non-denormals.
+            mantissa, exponent = math.frexp(val)
+            sign = True if mantissa < 0 else False
+            mantissa = -mantissa * 2 if sign else mantissa * 2
+            exponent -= 1
+
+            # Check if the number is finite, nonzero and not denormal, otherwise skip the check.
+            if not (math.isfinite(val) and val != 0 and exponent > DOUBLE_MIN_EXP):
+                continue
+
+            # Check if exponent is representable.
+            if (cls.typeExponentOffset + exponent) > cls.typeExponentMax or (cls.typeExponentOffset + exponent) < 0:
+                return False
+
+            # Check if mantissa is representable. Implicit assumption is that cls.typeMantissa < 52 (like in FP64)
+            truncated_mantissa = 1 + math.floor((2**cls.typeMantissa) * (mantissa - 1)) / (2**cls.typeMantissa)
+            if math.fabs(truncated_mantissa - mantissa) > 0.0:
+                return False
+
+        return True
+
+
 class Pointer(BaseType[Optional[str], _PointerType]):
     """Represents a C Pointer type to an underlying BaseType data type
     """

Deeploy/CommonExtensions/DataTypes.py

@@ -25,7 +25,7 @@
 
 from typing import Tuple, Type
 
-from Deeploy.AbstractDataTypes import IntegerImmediate
+from Deeploy.AbstractDataTypes import FloatImmediate, IntegerImmediate
 
 
 class int8_t(IntegerImmediate):
@@ -76,10 +76,26 @@ class uint64_t(IntegerImmediate):
     signed = False
 
 
+# BFloat16 in PULP systems
+class float16alt(FloatImmediate):
+    typeName = "float16alt"
+    typeWidth = 16
+    typeMantissa = 7
+    typeExponent = 8
+
+
+class float32(FloatImmediate):
+    typeName = "float"
+    typeWidth = 32
+    typeMantissa = 23
+    typeExponent = 8
+
+
 SignedIntegerDataTypes: Tuple[Type[IntegerImmediate], ...] = (int8_t, int16_t, int32_t, int64_t)
 UnsignedIntegerDataTypes: Tuple[Type[IntegerImmediate], ...] = (uint8_t, uint16_t, uint32_t, uint64_t)
 IntegerDataTypes: Tuple[Type[IntegerImmediate], ...] = (sorted((
     *SignedIntegerDataTypes,
     *UnsignedIntegerDataTypes,
 ),
                                                                key = lambda _type: _type.typeWidth))
+FloatDataTypes: Tuple[Type[FloatImmediate], ...] = (float16alt, float32)

Deeploy/DeeployTypes.py

@@ -45,7 +45,7 @@
 from onnx.external_data_helper import convert_model_to_external_data
 from ortools.constraint_solver.pywrapcp import IntVar
 
-from .AbstractDataTypes import BaseType, IntegerImmediate, Pointer, PointerClass, Struct, VoidType
+from .AbstractDataTypes import BaseType, FloatImmediate, IntegerImmediate, Pointer, PointerClass, Struct, VoidType
 
 Shape = TypeVar("Shape", bound = Any)
 SubGraph = List[gs.Node]
@@ -1903,11 +1903,16 @@ def _broadcastInteger(ty: Type[IntegerImmediate]):
             else:
                 return np.dtype(getattr(np, "uint" + str(ty.typeWidth)))
 
+        def _broadcastFloat(ty: Type[FloatImmediate]):
+            return np.dtype(getattr(np, "double"))
+
         if issubclass(ty, Pointer) and hasattr(ty, "referencedType"):
             if issubclass(ty.referencedType, IntegerImmediate):
                 return _broadcastInteger(ty.referencedType)
         elif issubclass(ty, IntegerImmediate):
             return _broadcastInteger(ty)
+        elif issubclass(ty, FloatImmediate):
+            return _broadcastFloat(ty)
 
         return None
 

Deeploy/Targets/Generic/Bindings.py

@@ -30,17 +30,18 @@
 from Deeploy.AbstractDataTypes import PointerClass
 from Deeploy.CommonExtensions.CodeTransformationPasses.MemoryAllocation import ArgumentStructGeneration, \
     MemoryManagementGeneration, MemoryPassthroughGeneration
-from Deeploy.CommonExtensions.DataTypes import IntegerDataTypes, SignedIntegerDataTypes, int8_t, int32_t, uint8_t
+from Deeploy.CommonExtensions.DataTypes import FloatDataTypes, IntegerDataTypes, SignedIntegerDataTypes, int8_t, \
+    int32_t, uint8_t
 from Deeploy.DeeployTypes import CodeTransformation, NodeBinding
 from Deeploy.FutureExtension.CodeTransformationPasses.FutureCodeTransformation import FutureGeneration
 from Deeploy.Targets.Generic.Templates import AddTemplate, ConcatTemplate, ConvTemplate, DebugPrintTemplate, \
-    DummyTemplate, DWConvTemplate, GatherTemplate, GemmTemplate, IntegerDivTemplate, ITAMaxTemplate, \
+    DummyTemplate, DWConvTemplate, FloatAddTemplate, GatherTemplate, GemmTemplate, IntegerDivTemplate, ITAMaxTemplate, \
     ITAPartialMaxTemplate, MatMulTemplate, MaxPoolTemplate, MulTemplate, PadTemplate, ReduceMeanTemplate, \
     ReduceSumTemplate, RequantShiftTemplate, ReshapeTemplate, RQIntegerDivTemplate, RQSiGELUTemplate, SliceTemplate, \
     TransposeTemplate, iGELUTemplate, iLayernormTemplate, iRMSNormTemplate, iSoftmaxTemplate
 from Deeploy.Targets.Generic.TypeCheckers import AddChecker, ConcatChecker, ConvChecker, DebugPrintChecker, \
-    DummyChecker, GatherChecker, GELUChecker, GEMMChecker, IntegerDivChecker, MatMulChecker, MaxPoolChecker, \
-    MulChecker, PadChecker, ReduceMeanChecker, ReduceSumChecker, RequantShiftChecker, ReshapeChecker, \
+    DummyChecker, FloatAddChecker, GatherChecker, GELUChecker, GEMMChecker, IntegerDivChecker, MatMulChecker, \
+    MaxPoolChecker, MulChecker, PadChecker, ReduceMeanChecker, ReduceSumChecker, RequantShiftChecker, ReshapeChecker, \
     RQIntegerDivChecker, SliceChecker, SoftmaxChecker, TransposeChecker, iLayerNormChecker
 
 BasicTransformer = CodeTransformation([ArgumentStructGeneration(), MemoryManagementGeneration(), FutureGeneration()])
@@ -65,6 +66,9 @@
                 AddTemplate.referenceTemplate, BasicTransformer)
     for type1 in IntegerDataTypes
     for type2 in IntegerDataTypes
+] + [
+    NodeBinding(FloatAddChecker([PointerClass(type), PointerClass(type)], [PointerClass(type)]),
+                FloatAddTemplate.referenceTemplate, BasicTransformer) for type in FloatDataTypes
 ]
 
 BasicConv1DBinding = NodeBinding(ConvChecker([PointerClass(int8_t), PointerClass(int8_t)], [PointerClass(int32_t)]),

Deeploy/Targets/Generic/Templates/FloatAddTemplate.py

@@ -0,0 +1,50 @@
+# ----------------------------------------------------------------------
+#
+# File: FloatAddTemplate.py
+#
+# Last edited: 15.12.2021
+#
+# Copyright (C) 2021, ETH Zurich and University of Bologna.
+#
+# Author: Moritz Scherer, ETH Zurich
+#
+# ----------------------------------------------------------------------
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the License); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an AS IS BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, List, Tuple
+
+from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation
+
+
+class _FloatAddTemplate(NodeTemplate):
+
+    def alignToContext(self, ctxt: NetworkContext,
+                       operatorRepresentation: OperatorRepresentation) -> Tuple[NetworkContext, Dict, List[str]]:
+
+        data_in_1 = ctxt.lookup(operatorRepresentation['data_in_1'])
+        data_in_2 = ctxt.lookup(operatorRepresentation['data_in_2'])
+        data_out = ctxt.lookup(operatorRepresentation['data_out'])
+
+        return ctxt, operatorRepresentation, []
+
+
+referenceTemplate = _FloatAddTemplate("""
+// Add (Name: ${nodeName}, Op: ${nodeOp})
+BEGIN_SINGLE_CORE
+    for (uint32_t i=0;i<${size};i++){
+        ${data_out}[i] = ${data_in_1}[i] + ${data_in_2}[i];
+    }
+END_SINGLE_CORE
+""")

Deeploy/Targets/Generic/TypeCheckers.py

@@ -125,6 +125,20 @@ def _inferSignedness(self, inputs: List[VariableBuffer],
             return [False]
 
 
+class FloatAddChecker(SignPropTypeChecker):
+
+    def __init__(self, input_types: Sequence[Type[Pointer]], output_types: Sequence[Type[Pointer]]):
+        super().__init__(input_types, output_types)
+
+    def _inferNumLevels(self, inputs: List[VariableBuffer],
+                        operatorRepresentation: OperatorRepresentation) -> List[int]:
+        return [inputs[0].nLevels + inputs[1].nLevels]
+
+    def _inferSignedness(self, inputs: List[VariableBuffer],
+                         operatorRepresentation: OperatorRepresentation) -> List[bool]:
+        return [True]
+
+
 class GatherChecker(SignPropTypeChecker):
 
     def __init__(self, input_types: Sequence[Type[Pointer]], output_types: Sequence[Type[Pointer]]):

DeeployTest/Tests/FloatAdder/network.onnx

@@ -0,0 +1,30 @@
+onnxruntime.transformers1.16.1:¨

+(
+input
+onnx::Add_1outputAdd_0"Addtorch-jit-exportZ
+input
+

+
+Z
+onnx::Add_1
+

+
+b
+output
+

+
+j
+output
+

+
+B
+B
+
+ai.onnx.mlB
+ai.onnx.training
B
+com.ms.internal.nhwcB
+ai.onnx.preview.training
B
+com.microsoft
B
+com.microsoft.experimental
B
+com.microsoft.nchwc
B
+org.pytorch.aten

DeeployTest/generateNetwork.py

@@ -76,9 +76,9 @@
         test_inputs, test_outputs, graph = generateDebugConfig(inputs, outputs, activations, graph)
 
     else:
-        # Load as int64 and infer types later
-        test_inputs = [inputs[x].reshape(-1).astype(np.int64) for x in inputs.files]
-        test_outputs = [outputs[x].reshape(-1).astype(np.int64) for x in outputs.files]
+        # Load as float64 and infer types later
+        test_inputs = [inputs[x].reshape(-1).astype(np.float64) for x in inputs.files]
+        test_outputs = [outputs[x].reshape(-1).astype(np.float64) for x in outputs.files]
 
         # WIESEP: Hack to get CI running because only one specific array is used
         if "WaveFormer" in args.dir:

DeeployTest/testTypes.py

@@ -28,7 +28,7 @@
 import pytest
 
 from Deeploy.AbstractDataTypes import PointerClass, StructClass
-from Deeploy.CommonExtensions.DataTypes import IntegerDataTypes, int8_t, int16_t, int32_t
+from Deeploy.CommonExtensions.DataTypes import IntegerDataTypes, bfloat16, float32, int8_t, int16_t, int32_t
 from Deeploy.DeeployTypes import ConstantBuffer, NetworkContext, StructBuffer, TransientBuffer, VariableBuffer
 
 
@@ -93,6 +93,32 @@ def testImmediatePromotion():
     return True
 
 
+def testImmediatePromotionFloat():
+    with pytest.raises(Exception):
+        _ = bfloat16(0.1)
+        _ = bfloat16(7777777.0)
+        _ = bfloat16(0.2)
+        _ = float32(77777777.0)
+        _ = float32(0.0000800006853044033050537109375)
+        c = bfloat16(7777777)
+    a = bfloat16(12.375)
+    b = bfloat16(0.5)
+    c = float32(7777777.0)
+    d = float32(77777776.0)
+
+    e = float32(0.0000900006853044033050537109375)
+
+    _ = bfloat16(0.000079631805419921875)
+
+    with pytest.raises(Exception):
+        _ = bfloat16(c)
+        _ = bfloat16(d)
+        _ = bfloat16(e)
+        _ = bfloat16(0.000079631805419921885)
+
+    return True
+
+
 def generateTestStruct() -> StructClass:
     testStructType = {"f1": int32_t, "f2": int8_t}
     s1 = StructClass("s2", testStructType)
@@ -223,6 +249,7 @@ def testPointerTypeEquivalence():
     testImmediateSerialization()
     testImmediatePromotion()
     testImmediateTypeEquivalence()
+    testImmediatePromotionFloat()
 
     testStructSerialization()
     testStructPromotion()

DeeployTest/testUtils/typeMapping.py

@@ -29,7 +29,7 @@
 import numpy as np
 
 from Deeploy.AbstractDataTypes import PointerClass
-from Deeploy.CommonExtensions.DataTypes import IntegerDataTypes, int8_t
+from Deeploy.CommonExtensions.DataTypes import FloatDataTypes, IntegerDataTypes, int8_t
 
 offsetType = namedtuple("offsetType", ("type", "offset"))
 
@@ -68,25 +68,26 @@ def inferInputType(_input: np.ndarray,
         print(f"Warning: Empty input array for type inference for {_input}!")
         return [(defaultType, defaultOffset)]
 
-    if not isInteger(_input):
-        raise Exception("Deeploy currently only handles integer types!")
-
     if signProp is None:
         signProp = False
 
     signedPlatformTypes = [_type for _type in IntegerDataTypes if _type.typeMin < 0]
 
     matchingTypes = []
 
-    if signProp and isUnsigned(_input):
+    if signProp and isUnsigned(_input) and isInteger(_input):
         for _type in sorted(signedPlatformTypes, key = lambda x: x.typeWidth):
             signPropOffset = (2**(_type.typeWidth - 1))
             if _type.checkPromotion(_input - signPropOffset):
                 matchingTypes.append(offsetType(PointerClass(_type), signPropOffset))
-    else:
+    elif isInteger(_input):
         for _type in sorted(IntegerDataTypes, key = lambda x: x.typeWidth):
             if _type.checkPromotion(_input):
                 matchingTypes.append(offsetType(PointerClass(_type), 0))
+    else:
+        for _type in sorted(FloatDataTypes, key = lambda x: x.typeWidth):
+            if _type.checkPromotion(_input):
+                matchingTypes.append(offsetType(PointerClass(_type), 0))
 
     if matchingTypes == []:
         raise Exception("Could not find a matching type!")

docs/install.md

@@ -83,8 +83,8 @@ To get started with Deeploy, you can run any of the regression tests in `deeploy
 For example, you can run
 
 ```
-cd deeploytest
-python testRunner_generic -t Tests/simpleRegression
+cd DeeployTest
+python testRunner_generic.py -t Tests/simpleRegression
 ```
 
 to run the `simpleRegression` test on your workstation. Various other tests are available and compatibility between tests and platforms is tested in the `.gitlab-ci.yml` file.