[WIP] Build line filter fix (#8343)

* Remove 'allocations' from BuildLineSerializer - Expensive to have a "many" serializer automatically used - Adjust existing tables accordingly - Fetch on demand * WIP: Add some unit tests * Adjust BuildLine queryset annotation - Multi-level annotation proves to be very expensive - Reduce complexity, save a bunch of time on queries - Remove 'total_allocated_stock' field - Adjust API query filter * Optimize query by deferring certain fields * Further query refinements * Bump API version
inventree · Oct 25, 2024 · 6be6c4b · 6be6c4b
1 parent 075b627
commit 6be6c4b
Show file tree

Hide file tree

Showing 6 changed files with 120 additions and 45 deletions.
diff --git a/src/backend/InvenTree/InvenTree/api_version.py b/src/backend/InvenTree/InvenTree/api_version.py
@@ -1,13 +1,16 @@
 """InvenTree API version information."""
 
 # InvenTree API version
-INVENTREE_API_VERSION = 271
+INVENTREE_API_VERSION = 272
 
 """Increment this API version number whenever there is a significant change to the API that any clients need to know about."""
 
 
 INVENTREE_API_TEXT = """
 
+v272 - 2024-10-25 : https://github.com/inventree/InvenTree/pull/8343
+    - Adjustments to BuildLine API serializers
+
 v271 - 2024-10-22 : https://github.com/inventree/InvenTree/pull/8331
     - Fixes for SalesOrderLineItem endpoints
 

diff --git a/src/backend/InvenTree/build/api.py b/src/backend/InvenTree/build/api.py
@@ -370,10 +370,10 @@ def filter_available(self, queryset, name, value):
         To determine this, we need to know:
 
         - The quantity required for each BuildLine
-        - The quantity available for each BuildLine
+        - The quantity available for each BuildLine (including variants and substitutes)
         - The quantity allocated for each BuildLine
         """
-        flt = Q(quantity__lte=F('total_available_stock') + F('allocated'))
+        flt = Q(quantity__lte=F('allocated') + F('available_stock') + F('available_substitute_stock') + F('available_variant_stock'))
 
         if str2bool(value):
             return queryset.filter(flt)
@@ -399,10 +399,13 @@ def get_source_build(self) -> Build:
     def get_queryset(self):
         """Override queryset to select-related and annotate"""
         queryset = super().get_queryset()
-        source_build = self.get_source_build()
-        queryset = build.serializers.BuildLineSerializer.annotate_queryset(queryset, build=source_build)
 
-        return queryset
+        if not hasattr(self, 'source_build'):
+            self.source_build = self.get_source_build()
+
+        source_build = self.source_build
+
+        return build.serializers.BuildLineSerializer.annotate_queryset(queryset, build=source_build)
 
 
 class BuildLineList(BuildLineEndpoint, DataExportViewMixin, ListCreateAPI):
@@ -446,15 +449,17 @@ class BuildLineList(BuildLineEndpoint, DataExportViewMixin, ListCreateAPI):
     def get_source_build(self) -> Build | None:
         """Return the target build for the BuildLine queryset."""
 
+        source_build = None
+
         try:
             build_id = self.request.query_params.get('build', None)
             if build_id:
-                build = Build.objects.get(pk=build_id)
-                return build
+                source_build = Build.objects.filter(pk=build_id).first()
         except (Build.DoesNotExist, AttributeError, ValueError):
             pass
 
-        return None
+        return source_build
+
 
 class BuildLineDetail(BuildLineEndpoint, RetrieveUpdateDestroyAPI):
     """API endpoint for detail view of a BuildLine object."""

diff --git a/src/backend/InvenTree/build/serializers.py b/src/backend/InvenTree/build/serializers.py
@@ -1279,7 +1279,9 @@ class Meta:
             'bom_item_detail',
             'part_detail',
             'quantity',
-            'allocations',
+
+            # Build detail fields
+            'build_reference',
 
             # BOM item detail fields
             'reference',
@@ -1303,7 +1305,6 @@ class Meta:
             'available_stock',
             'available_substitute_stock',
             'available_variant_stock',
-            'total_available_stock',
             'external_stock',
 
             # Extra fields only for data export
@@ -1317,6 +1318,9 @@ class Meta:
             'allocations',
         ]
 
+    # Build info fields
+    build_reference = serializers.CharField(source='build.reference', label=_('Build Reference'), read_only=True)
+
     # Part info fields
     part = serializers.PrimaryKeyRelatedField(source='bom_item.sub_part', label=_('Part'), many=False, read_only=True)
     part_name = serializers.CharField(source='bom_item.sub_part.name', label=_('Part Name'), read_only=True)
@@ -1340,9 +1344,17 @@ class Meta:
     bom_item = serializers.PrimaryKeyRelatedField(label=_('BOM Item'), read_only=True)
 
     # Foreign key fields
-    bom_item_detail = part_serializers.BomItemSerializer(source='bom_item', many=False, read_only=True, pricing=False)
+    bom_item_detail = part_serializers.BomItemSerializer(
+        source='bom_item',
+        many=False,
+        read_only=True,
+        pricing=False,
+        substitutes=False,
+        sub_part_detail=False,
+        part_detail=False
+    )
+
     part_detail = part_serializers.PartBriefSerializer(source='bom_item.sub_part', many=False, read_only=True, pricing=False)
-    allocations = BuildItemSerializer(many=True, read_only=True)
 
     # Annotated (calculated) fields
     allocated = serializers.FloatField(
@@ -1360,15 +1372,10 @@ class Meta:
         read_only=True
     )
 
-    available_stock = serializers.FloatField(
-        label=_('Available Stock'),
-        read_only=True
-    )
-
+    external_stock = serializers.FloatField(read_only=True, label=_('External Stock'))
+    available_stock = serializers.FloatField(read_only=True, label=_('Available Stock'))
     available_substitute_stock = serializers.FloatField(read_only=True, label=_('Available Substitute Stock'))
     available_variant_stock = serializers.FloatField(read_only=True, label=_('Available Variant Stock'))
-    total_available_stock = serializers.FloatField(read_only=True, label=_('Total Available Stock'))
-    external_stock = serializers.FloatField(read_only=True, label=_('External Stock'))
 
     @staticmethod
     def annotate_queryset(queryset, build=None):
@@ -1390,14 +1397,13 @@ def annotate_queryset(queryset, build=None):
             'build',
             'bom_item',
             'bom_item__part',
-            'bom_item__part__pricing_data',
             'bom_item__sub_part',
-            'bom_item__sub_part__pricing_data',
         )
 
         # Pre-fetch related fields
         queryset = queryset.prefetch_related(
-            'bom_item__sub_part__tags',
+            'allocations',
+
             'bom_item__sub_part__stock_items',
             'bom_item__sub_part__stock_items__allocations',
             'bom_item__sub_part__stock_items__sales_order_allocations',
@@ -1406,21 +1412,58 @@ def annotate_queryset(queryset, build=None):
             'bom_item__substitutes__part__stock_items',
             'bom_item__substitutes__part__stock_items__allocations',
             'bom_item__substitutes__part__stock_items__sales_order_allocations',
+        )
 
-            'allocations',
-            'allocations__stock_item',
-            'allocations__stock_item__part',
-            'allocations__stock_item__location',
-            'allocations__stock_item__location__tags',
-            'allocations__stock_item__supplier_part',
-            'allocations__stock_item__supplier_part__part',
-            'allocations__stock_item__supplier_part__supplier',
-            'allocations__stock_item__supplier_part__manufacturer_part',
-            'allocations__stock_item__supplier_part__manufacturer_part__manufacturer',
+        # Defer expensive fields which we do not need for this serializer
+
+        queryset = queryset.defer(
+            'build__lft',
+            'build__rght',
+            'build__level',
+            'build__tree_id',
+            'build__destination',
+            'build__take_from',
+            'build__completed_by',
+            'build__issued_by',
+            'build__sales_order',
+            'build__parent',
+            'build__notes',
+            'build__metadata',
+            'build__responsible',
+            'build__barcode_data',
+            'build__barcode_hash',
+            'build__project_code',
+        ).defer(
+            'bom_item__metadata'
+        ).defer(
+            'bom_item__part__lft',
+            'bom_item__part__rght',
+            'bom_item__part__level',
+            'bom_item__part__tree_id',
+            'bom_item__part__tags',
+            'bom_item__part__notes',
+            'bom_item__part__variant_of',
+            'bom_item__part__revision_of',
+            'bom_item__part__creation_user',
+            'bom_item__part__bom_checked_by',
+            'bom_item__part__default_supplier',
+            'bom_item__part__responsible_owner',
+        ).defer(
+            'bom_item__sub_part__lft',
+            'bom_item__sub_part__rght',
+            'bom_item__sub_part__level',
+            'bom_item__sub_part__tree_id',
+            'bom_item__sub_part__tags',
+            'bom_item__sub_part__notes',
+            'bom_item__sub_part__variant_of',
+            'bom_item__sub_part__revision_of',
+            'bom_item__sub_part__creation_user',
+            'bom_item__sub_part__bom_checked_by',
+            'bom_item__sub_part__default_supplier',
+            'bom_item__sub_part__responsible_owner',
         )
 
         # Annotate the "allocated" quantity
-        # Difficulty: Easy
         queryset = queryset.annotate(
             allocated=Coalesce(
                 Sum('allocations__quantity'), 0,
@@ -1448,7 +1491,6 @@ def annotate_queryset(queryset, build=None):
         )
 
         # Annotate the "on_order" quantity
-        # Difficulty: Medium
         queryset = queryset.annotate(
             on_order=part.filters.annotate_on_order_quantity(reference=ref),
         )
@@ -1511,12 +1553,4 @@ def annotate_queryset(queryset, build=None):
             )
         )
 
-        # Annotate with the 'total available stock'
-        queryset = queryset.annotate(
-            total_available_stock=ExpressionWrapper(
-                F('available_stock') + F('available_substitute_stock') + F('available_variant_stock'),
-                output_field=FloatField(),
-            )
-        )
-
         return queryset
diff --git a/src/backend/InvenTree/build/test_api.py b/src/backend/InvenTree/build/test_api.py
@@ -7,7 +7,7 @@
 from rest_framework import status
 
 from part.models import Part, BomItem
-from build.models import Build, BuildItem
+from build.models import Build, BuildItem, BuildLine
 from stock.models import StockItem
 
 from build.status_codes import BuildStatus
@@ -1471,3 +1471,29 @@ def test_valid_scraps(self):
             output.refresh_from_db()
             self.assertEqual(output.status, StockStatus.REJECTED)
             self.assertFalse(output.is_building)
+
+
+class BuildLineTests(BuildAPITest):
+    """Unit tests for the BuildLine API endpoints."""
+
+    def test_filter_available(self):
+        """Filter BuildLine objects by 'available' status."""
+
+        url = reverse('api-build-line-list')
+
+        # First *all* BuildLine objects
+        response = self.get(url)
+        self.assertEqual(len(response.data), BuildLine.objects.count())
+
+        # Filter by 'available' status
+        # Note: The max_query_time is bumped up here, as postgresql backend has some strange issues (only during testing)
+        response = self.get(url, data={'available': True}, max_query_time=15)
+        n_t = len(response.data)
+        self.assertGreater(n_t, 0)
+
+        # Note: The max_query_time is bumped up here, as postgresql backend has some strange issues (only during testing)
+        response = self.get(url, data={'available': False}, max_query_time=15)
+        n_f = len(response.data)
+        self.assertGreater(n_f, 0)
+
+        self.assertEqual(n_t + n_f, BuildLine.objects.count())
diff --git a/src/backend/InvenTree/part/serializers.py b/src/backend/InvenTree/part/serializers.py
@@ -1599,6 +1599,7 @@ def __init__(self, *args, **kwargs):
         part_detail = kwargs.pop('part_detail', False)
         sub_part_detail = kwargs.pop('sub_part_detail', True)
         pricing = kwargs.pop('pricing', True)
+        substitutes = kwargs.pop('substitutes', True)
 
         super().__init__(*args, **kwargs)
 
@@ -1608,6 +1609,9 @@ def __init__(self, *args, **kwargs):
         if not sub_part_detail:
             self.fields.pop('sub_part_detail', None)
 
+        if not substitutes:
+            self.fields.pop('substitutes', None)
+
         if not pricing:
             self.fields.pop('pricing_min', None)
             self.fields.pop('pricing_max', None)

diff --git a/src/backend/InvenTree/templates/js/translated/build.js b/src/backend/InvenTree/templates/js/translated/build.js
@@ -2502,7 +2502,10 @@ function renderBuildLineAllocationTable(element, build_line, options={}) {
 
     // Load the allocation items into the table
     sub_table.bootstrapTable({
-        data: build_line.allocations,
+        url: '{% url "api-build-item-list" %}',
+        queryParams: {
+            build_line: build_line.pk,
+        },
         showHeader: false,
         columns: [
             {