Resolve issues with schema and ArangoDB. Verify that arangoimport wor…

…ks as intended from the command line. Code is now largely abstracted, which should make extending platform support easier, though there are still likely issues with using alternative DB config files and/or importing foreign configurations.

data_models/base.py

@@ -43,7 +43,7 @@ class IndalekoBaseModel(BaseModel):
     '''
     def serialize(self) -> Dict[str, Any]:
         '''Serialize the object to a dictionary'''
-        return self.model_dump(exclude_unset=True)
+        return self.model_dump(exclude_unset=True, exclude_none=True)
 
     @classmethod
     def deserialize(cls: Type[T], data : Dict[str, Any]) -> T:
@@ -98,7 +98,7 @@ def test_model_main(cls : Type[T]) -> None:
         data = cls.get_example()
         ic(data)
         ic(dir(data))
-        print(data.model_dump_json(indent=2))
+        print(data.model_dump_json(indent=2, exclude_unset=True, exclude_none=True))
         serial_data = data.serialize()
         data_check = cls.deserialize(serial_data)
         assert data_check == data

data_models/i_object.py

@@ -65,7 +65,7 @@ class IndalekoObjectDataModel(IndalekoBaseModel):
                         title='Size',
                         description='The size of the object in bytes.')
 
-    SemanticAttributes : List[IndalekoSemanticAttributeDataModel] \
+    SemanticAttributes : Union[List[IndalekoSemanticAttributeDataModel], None] \
         = Field(None,
                 title='SemanticAttributes',
                 description='The semantic attributes related to this object by the storage service.')

storage/recorders/base.py

@@ -162,7 +162,7 @@ def __init__(self : 'BaseStorageRecorder', **kwargs : dict) -> None:
         '''
         self.recorder_service = IndalekoServiceManager().register_service(
             service_name = self.get_recorder_service_name(),
-            service_id = self.get_recorder_service_uuid(),
+            service_id = str(self.get_recorder_service_uuid()),
             service_version = self.get_recorder_service_version(),
             service_description = self.get_recorder_service_description(),
             service_type = self.get_recorder_service_type(),

storage/recorders/local/local_base.py

@@ -30,7 +30,7 @@
 import sys
 import uuid
 
-from typing import Union
+from typing import Union, Callable, Any
 
 from icecream import ic
 
@@ -46,6 +46,7 @@
 from perf.perf_collector import IndalekoPerformanceDataCollector
 from perf.perf_recorder import IndalekoPerformanceDataRecorder
 from platforms.machine_config import IndalekoMachineConfig
+from utils.cli.base import IndalekoBaseCLI
 from utils.cli.data_models.cli_data import IndalekoBaseCliDataModel
 from utils.cli.runner import IndalekoCLIRunner
 from utils.decorators import type_check
@@ -120,25 +121,40 @@ def execute_command(command : str) -> None:
         logging.info('Command %s result: %d', command, result)
         print(f'Command {command} result: {result}')
 
+    class local_recorder_mixin(IndalekoBaseCLI.default_handler_mixin):
+        '''This is the mixin for the local recorder'''
+
+        @staticmethod
+        def load_machine_config(keys : dict[str, str]) -> IndalekoMachineConfig:
+            assert 'class' in keys, '(machine config) class must be specified'
+            return BaseLocalStorageRecorder.load_machine_config(keys)
+
+        @staticmethod
+        def get_additional_parameters(pre_parser):
+            '''This method is used to add additional parameters to the parser.'''
+            return BaseLocalStorageRecorder.get_additional_parameters(pre_parser)
+
+
     @staticmethod
     def local_run(keys: dict[str, str]) -> Union[dict, None]:
-        '''Run the collector'''
-        ic('local_run: ', keys)
-        exit(0)
+        '''Run the recorder'''
         args = keys['args'] # must be there.
         cli = keys['cli'] # must be there.
         config_data = cli.get_config_data()
         debug = hasattr(args, 'debug') and args.debug
         if debug:
             ic(config_data)
-        recorder_class = keys['parameters']['Class']
+        recorder_class = keys['parameters']['RecorderClass']
+        machine_config_class = keys['parameters']['MachineConfigClass']
+        # collector_class = keys['parameters']['CollectorClass'] # unused for now
         # recorders have the machine_id so they need to find the
         # matching machine configuration file.
         kwargs = {
             'machine_config': cli.handler_mixin.load_machine_config(
                 {
                     'machine_config_file' : str(Path(args.configdir) / args.machine_config),
-                    'offline' : args.offline
+                    'offline' : args.offline,
+                    'class' : machine_config_class
                 }
             ),
             'timestamp': config_data['Timestamp'],
@@ -162,10 +178,11 @@ def capture_performance(
             perf_data = IndalekoPerformanceDataCollector.measure_performance(
                 task_func,
                 source=IndalekoSourceIdentifierDataModel(
-                    Identifier=recorder.service_identifier,
-                    Version = recorder.service_version,
-                    Description=recorder.service_description),
-                description=recorder.service_description,
+                    Identifier=recorder.get_recorder_service_uuid(),
+                    Version = recorder.get_recorder_service_version(),
+                    Description=recorder.get_recorder_service_description()
+                ),
+                description=recorder.get_recorder_service_description(),
                 MachineIdentifier=uuid.UUID(kwargs['machine_config'].machine_id),
                 process_results_func=extract_counters,
                 input_file_name=str(Path(args.datadir) / args.inputfile),
@@ -219,10 +236,9 @@ def capture_performance(
     @staticmethod
     def local_recorder_runner(
         collector_class: BaseStorageCollector,
-        recorder_class : BaseStorageRecorder) -> None:
+        recorder_class : BaseStorageRecorder,
+        machine_config_class : IndalekoMachineConfig) -> None:
         '''This is the CLI handler for local storage collectors.'''
-        ic(collector_class.get_collector_platform_name())
-        ic(recorder_class.get_recorder_service_name())
         runner = IndalekoCLIRunner(
             cli_data=IndalekoBaseCliDataModel(
                 Service=recorder_class.get_recorder_service_name(),
@@ -231,8 +247,12 @@ def local_recorder_runner(
                     'svc' : collector_class.get_collector_service_name(),
                 }
             ),
-            handler_mixin=local_recorder_mixin,
-            Run=local_run,
-            RunParameters={'RecorderClass' : recorder_class}
+            handler_mixin=BaseLocalStorageRecorder.local_recorder_mixin,
+            Run=BaseLocalStorageRecorder.local_run,
+            RunParameters={
+                'CollectorClass' : collector_class,
+                'MachineConfigClass' : machine_config_class,
+                'RecorderClass' : recorder_class
+            }
         )
         runner.run()

storage/recorders/local/windows/recorder.py

@@ -335,12 +335,12 @@ def record(self) -> None:
             'platform' : self.platform,
             'service' : IndalekoWindowsLocalStorageRecorder.windows_local_recorder_name,
             'storage' : self.storage_description,
-            'collection' : IndalekoDBCollections.Indaleko_Relationship_Collection,
+            'collection' : IndalekoDBCollections.Indaleko_Object_Collection,
             'timestamp' : self.timestamp,
             'output_dir' : self.data_dir,
         }
         self.output_object_file = self.generate_output_file_name(**kwargs)
-        kwargs['collection'] = IndalekoDBCollections.Indaleko_Object_Collection
+        kwargs['collection'] = IndalekoDBCollections.Indaleko_Relationship_Collection
         self.output_edge_file = self.generate_output_file_name(**kwargs)
 
     @staticmethod
@@ -398,7 +398,7 @@ def bulk_upload_relationship_data(recorder : 'IndalekoWindowsLocalStorageRecorde
         raise NotImplementedError('bulk_upload_relationship_data must be implemented')
 
 
-class local_recorder_mixin(IndalekoBaseCLI.default_handler_mixin):
+class old_local_recorder_mixin(IndalekoBaseCLI.default_handler_mixin):
     '''This is the mixin for the local recorder'''
 
     @staticmethod
@@ -413,7 +413,7 @@ def get_additional_parameters(pre_parser):
         return BaseLocalStorageRecorder.get_additional_parameters(pre_parser)
 
 @staticmethod
-def local_run(keys: dict[str, str]) -> Union[dict, None]:
+def old_local_run(keys: dict[str, str]) -> Union[dict, None]:
     '''Run the collector'''
     args = keys['args'] # must be there.
     cli = keys['cli'] # must be there.
@@ -511,7 +511,7 @@ def capture_performance(
         capture_performance(recorder.bulk_upload_relationship_data)
 
 @staticmethod
-def local_recorder_runner(
+def old_local_recorder_runner(
     collector_class: BaseStorageCollector,
     recorder_class : BaseStorageRecorder,
     machine_config_class : IndalekoMachineConfig) -> None:
@@ -536,7 +536,7 @@ def local_recorder_runner(
 
 def main():
     '''This is the CLI handler for the Windows local storage collector.'''
-    local_recorder_runner(
+    BaseLocalStorageRecorder.local_recorder_runner(
         IndalekoWindowsLocalCollector,
         IndalekoWindowsLocalStorageRecorder,
         IndalekoWindowsMachineConfig

utils/cli/base.py

@@ -208,7 +208,6 @@ def setup_machine_config_parser(self) -> 'IndalekoBaseCLI':
             pre_args, _ = self.pre_parser.parse_known_args()
         if hasattr(pre_args, 'inputfile'):
             machine_id = self.config_data['InputFileKeys'].get('machine', None)
-            ic(f'setup_machine_config_parser: Machine ID: {machine_id}')
         self.config_data['MachineConfigChoices'] = self.handler_mixin.find_machine_config_files(self.config_data['ConfigDirectory'], pre_args.platform)
         default_machine_config_file = self.handler_mixin.get_default_file(
             self.config_data['ConfigDirectory'],