From 50d11c09d281ee6e6605e6405607b5b5d5be1e66 Mon Sep 17 00:00:00 2001 From: aelmiger <40243985+aelmiger@users.noreply.github.com> Date: Fri, 5 Apr 2024 15:57:23 +0200 Subject: [PATCH] Add global evaluators support (#8) --- .../job_description/dynamic_evaluators.md | 20 +++ .../job_description/job_configuration.md | 19 ++- syclops/preprocessing/preprocessor.py | 35 +++++- syclops/schema/base_schema.yaml | 21 ++++ syclops/utility/__init__.py | 11 +- syclops/utility/blender_utils.py | 38 +++++- syclops/utility/sampling_utils.py | 118 ++++-------------- 7 files changed, 157 insertions(+), 105 deletions(-) diff --git a/docs/docs/usage/job_description/dynamic_evaluators.md b/docs/docs/usage/job_description/dynamic_evaluators.md index d7f54c6..32e4ee9 100644 --- a/docs/docs/usage/job_description/dynamic_evaluators.md +++ b/docs/docs/usage/job_description/dynamic_evaluators.md @@ -37,3 +37,23 @@ The following table lists all dynamic evaluators and their parameters. | random_selection | ```[, , ...]``` | Samples a value from a list of steps in a random order. | | selection_asset | ```{library: , type: } ``` | Randomly picks an asset from the given library and type. | | selection_wildcard | ```{library: , wildcard: }``` | Randomly selects an asset from a library that matches the given wildcard pattern. | + + +## Referencing Global Evaluators + +In addition to the dynamic evaluators that are specific to each attribute, you can also reference global evaluators defined in the `global_evaluators` section of the job configuration. Global evaluators are evaluated once per frame and ensure that the same random value is used for all sensors within a single frame. + +To reference a global evaluator, use the syntax `$global.`. For example: + +```yaml +sensor: + syclops_sensor_camera: + - name: "main_camera" + gamma: $global.gamma + # ... + - name: "secondary_camera" + gamma: $global.gamma + # ... +``` + +In this example, both the `main_camera` and `secondary_camera` will use the same random value for `gamma` in each frame, as defined in the `global_evaluators` section. \ No newline at end of file diff --git a/docs/docs/usage/job_description/job_configuration.md b/docs/docs/usage/job_description/job_configuration.md index 8cc79ae..e3f5d50 100644 --- a/docs/docs/usage/job_description/job_configuration.md +++ b/docs/docs/usage/job_description/job_configuration.md @@ -12,6 +12,7 @@ Each job can be divided into the following sections: * **sensor**: Configuration of individual sensors and their corresponding outputs. * **postprocessing** - [*optional]*: Operations that are applied to the generated data after creation. * **textures** - [*optional]*: Dynamically generated textures that can be used in the scene. +* **global_evaluators**: Defines global evaluators that can be referenced by multiple plugins or sensors. === "general" @@ -238,4 +239,20 @@ Each job can be divided into the following sections: res: 8 octaves: 4 ``` - \ No newline at end of file + + +=== "global_evaluators" + + This section defines global evaluators that can be referenced by multiple plugins or sensor. These evaluators are evaluated once per frame and ensure that the same random value is used for all plugins/sensors within a single frame. This is useful for multiple cameras that should have the same random exposure value every frame. + + !!! tip + See [Dynamic Evaluators](dynamic_evaluators.md) for more information. + + **Example**: + ```yaml + global_evaluators: + gamma: + uniform: [0.8, 1.2] + exposure: + normal: [0, 1] + ``` diff --git a/syclops/preprocessing/preprocessor.py b/syclops/preprocessing/preprocessor.py index 8bec1e3..4a7350f 100644 --- a/syclops/preprocessing/preprocessor.py +++ b/syclops/preprocessing/preprocessor.py @@ -7,6 +7,8 @@ from PIL import Image from ruamel.yaml import YAML from syclops.preprocessing.texture_processor import process_texture +from syclops import utility + CATALOG_LIBRARY_KEY = "Preprocessed Assets" @@ -140,14 +142,43 @@ def create_textures(config: dict, catalog: dict, output_path: str): _add_to_catalog(catalog, texture_name, asset_value, output_path) -def _set_seed(config: dict, seed: int): - np.random.seed(config["seeds"]["numpy"] * seed) +def evaluate_global_evaluators(config: dict, global_evaluators: dict, catalog: dict): + """ + Evaluates global evaluators in the config. + """ + evaluated_global_evaluators = {} + for key, eval_value in global_evaluators.items(): + is_list = isinstance(eval_value, list) or hasattr(eval_value, "to_list") + if is_list: + raise ValueError("Global evaluators must be dictionaries.") + evaluated_global_evaluators[key] = { + "step": [ + utility.apply_sampling(eval_value, step, catalog) + for step in range(config["steps"]) + ] + } + + return evaluated_global_evaluators def preprocess(config_path: str, catalog_path: str, schema: str, output_path: str): config = read_yaml(config_path) catalog = read_yaml(catalog_path) schema = read_yaml(schema) + + # Evaluate global evaluators + global_evaluators = config.get("global_evaluators", {}) + evaluated_global_evaluators = evaluate_global_evaluators( + config, global_evaluators, catalog + ) + + # Replace global evaluator references in sensor configurations + for sensor_config in config["sensor"].values(): + for sensor in sensor_config: + for key, value in sensor.items(): + if isinstance(value, str) and value.startswith("$global."): + global_key = value[8:] + sensor[key] = evaluated_global_evaluators[global_key] try: validate(config, schema) print("Job Config is valid!") diff --git a/syclops/schema/base_schema.yaml b/syclops/schema/base_schema.yaml index e9e226f..47529ec 100644 --- a/syclops/schema/base_schema.yaml +++ b/syclops/schema/base_schema.yaml @@ -84,6 +84,27 @@ properties: - $ref: "#/definitions/keep_overlapp" - $ref: "#/definitions/random_rectangles" minItems: 1 + global_evaluators: + type: object + additionalProperties: + anyOf: + - type: number + - $ref: "#/definitions/sample_linear_vector" + - $ref: "#/definitions/sample_linear_number" + - $ref: "#/definitions/sample_normal_vector" + - $ref: "#/definitions/sample_normal_number" + - $ref: "#/definitions/sample_uniform_vector" + - $ref: "#/definitions/sample_uniform_number" + - $ref: "#/definitions/sample_step_number" + - $ref: "#/definitions/sample_step_vector" + - $ref: "#/definitions/sample_step_string" + - $ref: "#/definitions/sample_random_selection_number" + - $ref: "#/definitions/sample_random_selection_vector" + - $ref: "#/definitions/sample_random_selection_string" + - $ref: "#/definitions/sample_selection_folder" + - $ref: "#/definitions/sample_selection_asset" + - $ref: "#/definitions/sample_wildcard" + required: [steps, seeds, render_device, denoising_enabled] allOf: diff --git a/syclops/utility/__init__.py b/syclops/utility/__init__.py index e41ca72..c26512d 100644 --- a/syclops/utility/__init__.py +++ b/syclops/utility/__init__.py @@ -18,12 +18,11 @@ load_image, load_img_as_array, merge_objects, refresh_modifiers, render_visibility, resize_textures, set_active_collection, set_seeds, - show_all_modifiers) - from .sampling_utils import (eval_param, interpolate_img, sample_linear, - sample_normal, sample_random_selection, - sample_selection_asset, sample_selection_folder, - sample_step, sample_uniform, sample_wildcard) - + show_all_modifiers, eval_param) +from .sampling_utils import (sample_linear, + sample_normal, sample_random_selection, + sample_selection_asset, sample_selection_folder, + sample_step, sample_uniform, sample_wildcard, apply_sampling) from .general_utils import (AtomicYAMLWriter, create_folder, find_class_id_mapping,get_site_packages_path, get_module_path, hash_vector) diff --git a/syclops/utility/blender_utils.py b/syclops/utility/blender_utils.py index 3b8b4fb..e31c0b4 100644 --- a/syclops/utility/blender_utils.py +++ b/syclops/utility/blender_utils.py @@ -9,6 +9,7 @@ import coacd import numpy as np from mathutils import Matrix +from . import sampling_utils as su def apply_transform( @@ -443,6 +444,7 @@ def convex_decomposition( return convex_hulls + class DisjointSet: def __init__(self): self.parent = {} @@ -468,7 +470,7 @@ def get_clusters(self): clusters[root] = set() clusters[root].add(item) return list(clusters.values()) - + def find_cluster(self, item): if item not in self.parent: return None # Item not present in any cluster @@ -479,7 +481,8 @@ def find_cluster(self, item): if self.find(key) == root: cluster.add(key) return cluster - + + def run_coacd(quality: float, coacd_mesh: coacd.Mesh) -> list: """Run the COACD algorithm on a mesh with default parameters. @@ -638,7 +641,14 @@ def add_volume_attribute(obj: bpy.types.Object): bm = bmesh.new() bm.from_mesh(obj.data) - volume = float(bm.calc_volume()) * volume_conversion_factor + # Calculate the raw volume (without considering scaling) + raw_volume = float(bm.calc_volume()) + + # Calculate the scale factor (product of the scale on all axes) + scale_factor = obj.scale.x * obj.scale.y * obj.scale.z + # Adjust the volume for the object's scaling + volume = raw_volume * scale_factor * volume_conversion_factor + # Remove bmesh bm.free() bm = None @@ -789,3 +799,25 @@ def merge_objects(obj_list: list) -> bpy.types.Object: ): bpy.ops.object.join() return obj_list[0] + + +def eval_param(eval_params: Union[float, dict, str]) -> Union[float, list, str]: + """Convert a parameter to a discrete value with the help of a sample function. + + Args: + eval_params: Parameter to be evaluated. + + Returns: + Union[float, list, str]: Evaluated parameter. + """ + is_list = isinstance(eval_params, list) or hasattr(eval_params, "to_list") + eval_params = eval_params if is_list else [eval_params] + + curr_frame = bpy.context.scene.frame_current + catalog_string = bpy.data.scenes["Scene"]["catalog"] + catalog = pickle.loads(bytes(catalog_string, "latin1")) + + evaluated_param = list( + map(lambda x: su.apply_sampling(x, curr_frame, catalog), eval_params) + ) + return evaluated_param if is_list else evaluated_param[0] diff --git a/syclops/utility/sampling_utils.py b/syclops/utility/sampling_utils.py index 548e6f6..7e4bf8e 100644 --- a/syclops/utility/sampling_utils.py +++ b/syclops/utility/sampling_utils.py @@ -8,62 +8,10 @@ import glob import logging import numbers -import pickle from typing import List, Union - -import bpy import numpy as np -# from scipy.interpolate import RegularGridInterpolator - - -# def interpolate_img(img: np.ndarray, locations: np.ndarray) -> np.ndarray: -# """Get image value at a specific subpixel location using RegularGridInterpolator. - -# Args: -# img: Image as a numpy ndarray. -# locations: Locations where image value is needed. - -# Returns: -# np.ndarray: Interpolated image values. -# """ -# rows = np.arange(img.shape[0]) -# cols = np.arange(img.shape[1]) - -# pixels = (rows, cols) -# interpol_func = RegularGridInterpolator( -# pixels, -# img, -# method="linear", -# bounds_error=False, -# fill_value=0, -# ) -# return interpol_func(locations) -def interpolate_img(img: np.ndarray, locations: np.ndarray) -> np.ndarray: - """Get image value at a specific subpixel location using bilinear interpolation.""" - - # Extract x and y coordinates from locations - y, x = locations[:, 0], locations[:, 1] - - # Obtain the four surrounding integer coordinates - x1 = np.floor(x).astype(int) - x2 = np.clip(x1 + 1, 0, img.shape[1]-1) - y1 = np.floor(y).astype(int) - y2 = np.clip(y1 + 1, 0, img.shape[0]-1) - - # Compute the weights - w1, w2 = x - x1, y - y1 - - # Bilinear interpolation formula - values = ( - (1 - w1) * (1 - w2) * img[y1, x1] + - w1 * (1 - w2) * img[y1, x2] + - (1 - w1) * w2 * img[y2, x1] + - w1 * w2 * img[y2, x2] - ) - - return values - -def sample_normal(config: tuple) -> np.ndarray: + +def sample_normal(config: tuple, *args) -> np.ndarray: """Sample from a normal distribution. Args: @@ -77,7 +25,7 @@ def sample_normal(config: tuple) -> np.ndarray: return np.random.normal(mean, std) -def sample_uniform(config: tuple) -> np.ndarray: +def sample_uniform(config: tuple, *args) -> np.ndarray: """Sample from a uniform distribution. Args: @@ -91,35 +39,42 @@ def sample_uniform(config: tuple) -> np.ndarray: return np.random.uniform(min_val, max_val) -def sample_step(config: List[Union[float, List]]) -> Union[float, List]: +def sample_step( + config: List[Union[float, List]], curr_frame, *args +) -> Union[float, List]: """Sample a value based on the current frame in Blender. Args: config: List of values to be sampled. + curr_frame: Current frame in Blender. Returns: Union[float, List]: Sampled value. """ - curr_frame = bpy.context.scene.frame_current return config[curr_frame % len(config)] -def sample_linear(config: List[Union[float, List]]) -> Union[float, List]: +def sample_linear( + config: List[Union[float, List]], curr_frame, *args +) -> Union[float, List]: """Sample a linearly interpolated value based on the current frame. Args: config: List of values to be sampled. + curr_frame: Current frame in Blender. Returns: Union[float, List]: Sampled value. """ - curr_frame = bpy.context.scene.frame_current + start_val = np.array(config[0]) step_val = np.array(config[1]) return start_val + step_val * curr_frame -def sample_random_selection(config: List[Union[float, List]]) -> Union[float, List]: +def sample_random_selection( + config: List[Union[float, List]], *args +) -> Union[float, List]: """Randomly select a value from the given list. Args: @@ -131,7 +86,7 @@ def sample_random_selection(config: List[Union[float, List]]) -> Union[float, Li return config[np.random.randint(0, len(config))] -def sample_selection_folder(config: str) -> str: +def sample_selection_folder(config: str, *args) -> str: """Randomly select a file from a given folder. Args: @@ -144,26 +99,16 @@ def sample_selection_folder(config: str) -> str: return files_in_folder[np.random.randint(0, len(files_in_folder))] -def _get_catalog_from_scene() -> dict: - """Get the catalog dictionary from the Blender scene. - - Returns: - dict: Catalog dictionary. - """ - catalog_string = bpy.data.scenes["Scene"]["catalog"] - return pickle.loads(bytes(catalog_string, "latin1")) - - -def sample_selection_asset(config: dict) -> str: +def sample_selection_asset(config: dict, *args, catalog) -> str: """Randomly select an asset from a given catalog. Args: config: Dictionary containing the catalog name and asset type. + catalog: Dictionary containing the catalog. Returns: str: Path to the selected asset. """ - catalog = _get_catalog_from_scene() library = catalog[config["library"]]["assets"] asset_names = [ name for name, asset in library.items() if asset["type"] == config["type"] @@ -172,23 +117,23 @@ def sample_selection_asset(config: dict) -> str: return "{0}/{1}".format(config["library"], selected_asset) -def sample_wildcard(config: dict) -> str: +def sample_wildcard(config: dict, *args, catalog) -> str: """Randomly select an asset matching a given pattern from a catalog. Args: config: Dictionary containing the catalog name and pattern. + catalog: Dictionary containing the catalog. Returns: str: Path to the selected asset. """ - catalog = _get_catalog_from_scene() library = catalog[config["library"]]["assets"] asset_names = [name for name in library if fnmatch.fnmatch(name, config["pattern"])] selected_asset = asset_names[np.random.randint(0, len(asset_names))] return "{0}/{1}".format(config["library"], selected_asset) -def _sample(parameter): +def apply_sampling(parameter, curr_frame=None, catalog=None): sample_functions = [ "normal", "uniform", @@ -197,28 +142,15 @@ def _sample(parameter): "random_selection", "selection_folder", "selection_asset", + "wildcard", ] if isinstance(parameter, (numbers.Number, str)): return parameter for sample_func in sample_functions: if sample_func in parameter: - return eval("sample_{0}".format(sample_func))(parameter[sample_func]) + return eval(f"sample_{sample_func}")( + parameter[sample_func], curr_frame, catalog + ) logging.warning("Parameter {0} not supported format".format(parameter)) raise ValueError("Parameter {0} not supported format".format(parameter)) - - -def eval_param(eval_params: Union[float, dict, str]) -> Union[float, list, str]: - """Convert a parameter to a discrete value with the help of a sample function. - - Args: - eval_params: Parameter to be evaluated. - - Returns: - Union[float, list, str]: Evaluated parameter. - """ - is_list = isinstance(eval_params, list) or hasattr(eval_params, "to_list") - eval_params = eval_params if is_list else [eval_params] - - evaluated_param = list(map(_sample, eval_params)) - return evaluated_param if is_list else evaluated_param[0] \ No newline at end of file