From c31906c0086cf9e13cc3866bcd74ced3f72f4081 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Sun, 25 Sep 2022 00:27:27 -0700 Subject: [PATCH 01/17] Add more secret manager support --- feathr_project/feathr/secrets/abc.py | 18 ++++++++++ .../feathr/secrets/aws_secretmanager.py | 36 +++++++++++++++++++ feathr_project/feathr/spark_provider/_abc.py | 6 ++-- 3 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 feathr_project/feathr/secrets/abc.py create mode 100644 feathr_project/feathr/secrets/aws_secretmanager.py diff --git a/feathr_project/feathr/secrets/abc.py b/feathr_project/feathr/secrets/abc.py new file mode 100644 index 000000000..7b6e7178c --- /dev/null +++ b/feathr_project/feathr/secrets/abc.py @@ -0,0 +1,18 @@ +from abc import ABC, abstractmethod + +from typing import Any, Dict, List, Optional, Tuple + + +class SecretManagementClient(ABC): + """This is the abstract class for all the spark launchers. All the Spark launcher should implement those interfaces + """ + + @abstractmethod + def __init__(self, secret_namespace: str, secret_client) -> None: + pass + + @abstractmethod + def get_feathr_secret(self, secret_string: str): + """ + """ + pass diff --git a/feathr_project/feathr/secrets/aws_secretmanager.py b/feathr_project/feathr/secrets/aws_secretmanager.py new file mode 100644 index 000000000..96c6cc0ea --- /dev/null +++ b/feathr_project/feathr/secrets/aws_secretmanager.py @@ -0,0 +1,36 @@ +from loguru import logger +import json +from feathr.secrets.abc import SecretManagementClient + +from botocore.exceptions import ClientError + + +class AWSSecretManagerClient(SecretManagementClient): + def __init__(self, secret_name: str, secret_client): + self.secret_name = secret_name + if secret_client.meta.service_model.service_name != "secretsmanager": + raise RuntimeError("secret_client need to be a secretsmanager") + else: + self.secret_client = secret_client + + def get_feathr_secret(self, secret_string: str): + """Get Feathr Secrets from Azure Key Vault. Note that this function will replace '_' in `secret_string` with '-' since Azure Key Vault doesn't support it + """ + if self.secret_client is None: + raise RuntimeError( + "You need to initialize a boto3 secretmanager class and pass it to Feathr") + try: + get_secret_value_response = self.secret_client.get_secret_value( + SecretId=secret_string + ) + if 'SecretString' in get_secret_value_response: + secret = json.loads(get_secret_value_response['SecretString']) + return secret[secret_string] + else: + raise RuntimeError("Only string format is supported") + except KeyError as e: + logger.error( + f"Secret {secret_string} cannot be found in secretsmanager {self.secret_name}.") + raise e + except ClientError as e: + raise e diff --git a/feathr_project/feathr/spark_provider/_abc.py b/feathr_project/feathr/spark_provider/_abc.py index 2644f82fe..ff82e27ab 100644 --- a/feathr_project/feathr/spark_provider/_abc.py +++ b/feathr_project/feathr/spark_provider/_abc.py @@ -2,6 +2,7 @@ from typing import Any, Dict, List, Optional, Tuple + class SparkJobLauncher(ABC): """This is the abstract class for all the spark launchers. All the Spark launcher should implement those interfaces """ @@ -15,7 +16,6 @@ def upload_or_get_cloud_path(self, local_path_or_http_path: str): """ pass - @abstractmethod def submit_feathr_job(self, job_name: str, main_jar_path: str, main_class_name: str, arguments: List[str], reference_files_path: List[str], job_tags: Dict[str, str] = None, @@ -33,6 +33,7 @@ def submit_feathr_job(self, job_name: str, main_jar_path: str, main_class_name: properties (Dict[str, str]): Additional System Properties for the spark job """ pass + @abstractmethod def wait_for_completion(self, timeout_seconds: Optional[float]) -> bool: """Returns true if the job completed successfully @@ -52,8 +53,5 @@ def get_status(self) -> str: Returns: str: Status of the current job - - Returns: - str: _description_ """ pass From e96459ab45486adfbfe84ab67ecff55b026c391f Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Mon, 26 Sep 2022 02:37:08 -0700 Subject: [PATCH 02/17] Add abstract class --- .../feathr-configuration-and-env.md | 49 ++++++++++++++++++- feathr_project/feathr/client.py | 11 ++--- feathr_project/feathr/secrets/abc.py | 20 ++++++-- feathr_project/feathr/secrets/akv_client.py | 37 +++++++------- .../feathr/secrets/aws_secretmanager.py | 41 +++++++--------- .../feathr/utils/_envvariableutil.py | 49 ++++++++++--------- feathr_project/setup.py | 3 +- 7 files changed, 132 insertions(+), 78 deletions(-) diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index fd32fb2f6..2aec11d89 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -27,8 +27,53 @@ This allows end users to store the configuration in a secure way, say in Kuberne Feathr will get the configurations in the following order: 1. If the key is set in the environment variable, Feathr will use the value of that environment variable -2. If it's not set in the environment, then a value is retrieved from the feathr_config.yaml file with the same config key. -3. If it's not available in the feathr_config.yaml file, Feathr will try to retrieve the value from a key vault service. Currently only Azure Key Vault is supported. +2. If it's not set in the environment, then a value is retrieved from the `feathr_config.yaml` file with the same config key. +3. If it's not available in the `feathr_config.yaml` file, Feathr will try to retrieve the value from a key vault service. Currently both Azure Key Vault and AWS Secrets Manager are supported. + +# Using Secret Management Service in Feathr + +Feathr supports using a Secret Management service for all the credentials and environment variables. Currently the supported secret management services are Azure Key Vault and AWS Secrets Manager. + +In order to use those secret management service, there are two steps: + +Step 1: Tell Feathr which secret management service to use, and what is the corresponding namespace. + +If using Azure Key Vault: +```yaml +secrets: + azure_key_vault: + name: feathrazuretest3-kv +``` + +If using AWS Secret Manager, users should put the corresponding secret_id in the `feathr_config.yaml` section, like below, so that Feathr knows which secret_id to use to retrieve the required credentials. +```yaml +secrets: + aws_secrets_manager: + secret_id: feathrsecret_namespace +``` + +Step 2: Initialize a secret management client and pass it to Feathr. + +For Azure Key Vault: +```python + +SecretClient( + vault_url = f"https://{self.akv_name}.vault.azure.net", + credential=DefaultAzureCredential() + ) +``` + +For AWS Secrets Manager: +```python +client = botocore.session.get_session().create_client( + service_name='secretsmanager', + aws_access_key_id = '', + aws_secret_access_key= '', + region_name=region_name +) +cache_config = SecretCacheConfig() +cache = SecretCache( config = cache_config, client = client) +``` # A list of environment variables that Feathr uses diff --git a/feathr_project/feathr/client.py b/feathr_project/feathr/client.py index f21d37d23..9cf1e7108 100644 --- a/feathr_project/feathr/client.py +++ b/feathr_project/feathr/client.py @@ -52,17 +52,14 @@ class FeathrClient(object): config_path (str, optional): config path. See [Feathr Config Template](https://github.com/feathr-ai/feathr/blob/main/feathr_project/feathrcli/data/feathr_user_workspace/feathr_config.yaml) for more details. Defaults to "./feathr_config.yaml". local_workspace_dir (str, optional): set where is the local work space dir. If not set, Feathr will create a temporary folder to store local workspace related files. credential (optional): credential to access cloud resources, most likely to be the returned result of DefaultAzureCredential(). If not set, Feathr will initialize DefaultAzureCredential() inside the __init__ function to get credentials. - project_registry_tag (Dict[str, str]): adding tags for project in Feathr registry. This might be useful if you want to tag your project as deprecated, or allow certain customizations on project leve. Default is empty - - Raises: - RuntimeError: Fail to create the client since necessary environment variables are not set for Redis - client creation. + project_registry_tag (Dict[str, str]): adding tags for project in Feathr registry. This might be useful if you want to tag your project as deprecated, or allow certain customizations on project level. Default is empty. + secret_manager_client: the secret manager client initialized outside of Feathr. End users need to initialize the secret manager outside of Feathr and pass it to Feathr so Feathr can use it to get required secrets. """ - def __init__(self, config_path:str = "./feathr_config.yaml", local_workspace_dir: str = None, credential=None, project_registry_tag: Dict[str, str]=None): + def __init__(self, config_path:str = "./feathr_config.yaml", local_workspace_dir: str = None, credential=None, project_registry_tag: Dict[str, str]=None, secret_manager_client = None): self.logger = logging.getLogger(__name__) # Redis key separator self._KEY_SEPARATOR = ':' - self.envutils = _EnvVaraibleUtil(config_path) + self.envutils = _EnvVaraibleUtil(config_path, secret_manager_client) if local_workspace_dir: self.local_workspace_dir = local_workspace_dir else: diff --git a/feathr_project/feathr/secrets/abc.py b/feathr_project/feathr/secrets/abc.py index 7b6e7178c..7ae20ebe1 100644 --- a/feathr_project/feathr/secrets/abc.py +++ b/feathr_project/feathr/secrets/abc.py @@ -3,16 +3,28 @@ from typing import Any, Dict, List, Optional, Tuple -class SecretManagementClient(ABC): - """This is the abstract class for all the spark launchers. All the Spark launcher should implement those interfaces +class FeathrSecretsManagementClient(ABC): + """This is the abstract class for all the secrets management service, which are used to store the credentials that Feathr might use. """ @abstractmethod def __init__(self, secret_namespace: str, secret_client) -> None: + """Initialize the FeathrSecretsManagementClient class. + + Args: + secret_namespace (str): a namespace that Feathr needs to get secrets from. + For Azure Key Vault, it is something like the key vault name. + For AWS secrets manager, it is something like a secret name. + + secret_client: A client that will be used to retrieve Feathr secrets. + """ pass @abstractmethod - def get_feathr_secret(self, secret_string: str): - """ + def get_feathr_secret(self, secret_string: str) -> str: + """Get Feathr Secrets from a certain secret management service, such as Azure Key Vault or AWS Secrets Manager. + + Returns: + str: returned secret from secret management service """ pass diff --git a/feathr_project/feathr/secrets/akv_client.py b/feathr_project/feathr/secrets/akv_client.py index cdec01e12..53f2d3e31 100644 --- a/feathr_project/feathr/secrets/akv_client.py +++ b/feathr_project/feathr/secrets/akv_client.py @@ -1,31 +1,32 @@ from azure.keyvault.secrets import SecretClient -from azure.identity import DefaultAzureCredential from loguru import logger from azure.core.exceptions import ResourceNotFoundError +from feathr.secrets.abc import FeathrSecretsManagementClient -class AzureKeyVaultClient: - def __init__(self, akv_name: str): - self.akv_name = akv_name - self.secret_client = None - def get_feathr_akv_secret(self, secret_name: str): +class AzureKeyVaultClient(FeathrSecretsManagementClient): + def __init__(self, secret_namespace: str, secret_client: SecretClient = None): + if isinstance(secret_client, SecretClient): + self.secret_client = secret_client + else: + raise RuntimeError("You need to pass an azure.keyvault.secrets.SecretClient instance.") + + def get_feathr_secret(self, secret_name: str) -> str: """Get Feathr Secrets from Azure Key Vault. Note that this function will replace '_' in `secret_name` with '-' since Azure Key Vault doesn't support it Returns: - _type_: _description_ + str: returned secret from secret management service """ - if self.secret_client is None: - self.secret_client = SecretClient( - vault_url = f"https://{self.akv_name}.vault.azure.net", - credential=DefaultAzureCredential() - ) try: # replace '_' with '-' since Azure Key Vault doesn't support it - variable_replaced = secret_name.replace('_','-') #.upper() - logger.info('Fetching the secret {} from Key Vault {}.', variable_replaced, self.akv_name) + variable_replaced = secret_name.replace('_', '-') # .upper() + logger.info('Fetching the secret {} from Key Vault {}.', + variable_replaced, self.secret_client.vault_url) secret = self.secret_client.get_secret(variable_replaced) - logger.info('Secret {} fetched from Key Vault {}.', variable_replaced, self.akv_name) + logger.info('Secret {} fetched from Key Vault {}.', + variable_replaced, self.secret_client.vault_url) return secret.value - except ResourceNotFoundError as e: - logger.error(f"Secret {secret_name} cannot be found in Key Vault {self.akv_name}.") - raise \ No newline at end of file + except ResourceNotFoundError: + logger.error( + f"Secret {secret_name} cannot be found in Key Vault {self.secret_client.vault_url}.") + raise diff --git a/feathr_project/feathr/secrets/aws_secretmanager.py b/feathr_project/feathr/secrets/aws_secretmanager.py index 96c6cc0ea..9271aae34 100644 --- a/feathr_project/feathr/secrets/aws_secretmanager.py +++ b/feathr_project/feathr/secrets/aws_secretmanager.py @@ -1,36 +1,33 @@ from loguru import logger import json -from feathr.secrets.abc import SecretManagementClient - +from feathr.secrets.abc import FeathrSecretsManagementClient +import botocore from botocore.exceptions import ClientError +from aws_secretsmanager_caching.secret_cache import SecretCache + +class AWSSecretManagerClient(FeathrSecretsManagementClient): + def __init__(self, secret_namespace: str = None, secret_client: SecretCache = None): + self.secret_id = secret_namespace + self.secret_namespace = secret_namespace -class AWSSecretManagerClient(SecretManagementClient): - def __init__(self, secret_name: str, secret_client): - self.secret_name = secret_name - if secret_client.meta.service_model.service_name != "secretsmanager": - raise RuntimeError("secret_client need to be a secretsmanager") + # make sure secret_client is a SecretCache type + if not isinstance(secret_client, SecretCache): + raise RuntimeError( + "You need to pass a aws_secretsmanager_caching.secret_cache.SecretCache instance. Please refer to https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_cache-python.html for more details.") else: self.secret_client = secret_client def get_feathr_secret(self, secret_string: str): - """Get Feathr Secrets from Azure Key Vault. Note that this function will replace '_' in `secret_string` with '-' since Azure Key Vault doesn't support it + """Get Feathr Secrets from AWS Secrets manager. It's also recommended that the client passes a cache objects to reduce cost. + See more details here: https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_cache-python.html """ - if self.secret_client is None: - raise RuntimeError( - "You need to initialize a boto3 secretmanager class and pass it to Feathr") try: - get_secret_value_response = self.secret_client.get_secret_value( - SecretId=secret_string - ) - if 'SecretString' in get_secret_value_response: - secret = json.loads(get_secret_value_response['SecretString']) - return secret[secret_string] - else: - raise RuntimeError("Only string format is supported") + get_secret_value_response = self.secret_client.get_secret_string(self.secret_namespace) + # result is in str format, so we need to load it as a dict + secret = json.loads(get_secret_value_response) + return secret[secret_string] except KeyError as e: logger.error( - f"Secret {secret_string} cannot be found in secretsmanager {self.secret_name}.") - raise e - except ClientError as e: + f"Secret {secret_string} cannot be found in secretsmanager {self.secret_id}.") raise e diff --git a/feathr_project/feathr/utils/_envvariableutil.py b/feathr_project/feathr/utils/_envvariableutil.py index 8fcc85842..9d2863461 100644 --- a/feathr_project/feathr/utils/_envvariableutil.py +++ b/feathr_project/feathr/utils/_envvariableutil.py @@ -2,15 +2,27 @@ import yaml from loguru import logger from feathr.secrets.akv_client import AzureKeyVaultClient -from azure.core.exceptions import ResourceNotFoundError +from feathr.secrets.aws_secretmanager import AWSSecretManagerClient class _EnvVaraibleUtil(object): - def __init__(self, config_path): + def __init__(self, config_path: str, secret_manager_client): + """Initialize the environment variable utils client + + Args: + config_path (str): configuration path, if users want to use YAML to load all the configs + secret_manager_client: the secret manager client type. currently only Azure key vault and AWS secret manager is supported. + """ self.config_path = config_path # Set to none first to avoid invalid reference - self.akv_name = None - self.akv_name = self.get_environment_variable_with_default( 'secrets', 'azure_key_vault', 'name') - self.akv_client = AzureKeyVaultClient(self.akv_name) if self.akv_name else None + self.secret_manager_client = None + if self.get_environment_variable_with_default('secrets', 'azure_key_vault', 'name'): + self.secret_manager_client = AzureKeyVaultClient( + secret_namespace=self.get_environment_variable_with_default('secrets', 'azure_key_vault', 'name'), + secret_client=self.secret_manager_client) + elif self.get_environment_variable_with_default('secrets', 'aws_secrets_manager', 'secret_id'): + self.secret_manager_client = AWSSecretManagerClient( + secret_namespace=self.get_environment_variable_with_default('secrets', 'aws_secrets_manager', 'secret_id'), + secret_client=self.secret_manager_client) def get_environment_variable_with_default(self, *args): """Gets the environment variable for the variable key. @@ -48,22 +60,17 @@ def get_environment_variable_with_default(self, *args): yaml_layer = yaml_layer[arg] return yaml_layer except KeyError as exc: - logger.info("{} not found in the config file.", env_keyword) + logger.info( + "{} not found in the config file.", env_keyword) except yaml.YAMLError as exc: logger.warning(exc) - + # If it's not available in the feathr_config.yaml file, Feathr will try to retrieve the value from key vault - if self.akv_name: - try: - return self.akv_client.get_feathr_akv_secret(env_keyword) - except ResourceNotFoundError: - # print out warning message if cannot find the env variable in all the resources - logger.warning('Environment variable {} not found in environment variable, default YAML config file, or key vault service.', env_keyword) - return None + if self.secret_manager_client: + return self.secret_manager_client.get_feathr_secret(env_keyword) def get_environment_variable(self, variable_key): """Gets the environment variable for the variable key. - Args: variable_key: environment variable key that is used to retrieve the environment variable @@ -80,12 +87,6 @@ def get_environment_variable(self, variable_key): # If it's not available in the environment variable file, Feathr will try to retrieve the value from key vault logger.info(variable_key + ' is not set in the environment variables.') - - if self.akv_name: - try: - return self.akv_client.get_feathr_akv_secret(variable_key) - except ResourceNotFoundError: - # print out warning message if cannot find the env variable in all the resources - logger.warning('Environment variable {} not found in environment variable or key vault service.', variable_key) - return None - \ No newline at end of file + + if self.secret_manager_client: + return self.secret_manager_client.get_feathr_secret(variable_key) diff --git a/feathr_project/setup.py b/feathr_project/setup.py index e937f19c4..373709395 100644 --- a/feathr_project/setup.py +++ b/feathr_project/setup.py @@ -52,7 +52,8 @@ # https://github.com/Azure/azure-sdk-for-python/pull/22891 # using a version lower than that to workaround this issue. "azure-core<=1.22.1", - "typing_extensions>=4.2.0" + "typing_extensions>=4.2.0", + "aws-secretsmanager-caching>=1.1.1.5", ], tests_require=[ 'pytest', From 2d6c135e34cf700dc752efeb9cae1c9137f5bec8 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Tue, 27 Sep 2022 07:23:57 -0700 Subject: [PATCH 03/17] Update feathr-configuration-and-env.md --- .../how-to-guides/feathr-configuration-and-env.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/how-to-guides/feathr-configuration-and-env.md b/docs/how-to-guides/feathr-configuration-and-env.md index 2aec11d89..e52ca3706 100644 --- a/docs/how-to-guides/feathr-configuration-and-env.md +++ b/docs/how-to-guides/feathr-configuration-and-env.md @@ -56,15 +56,20 @@ Step 2: Initialize a secret management client and pass it to Feathr. For Azure Key Vault: ```python - -SecretClient( - vault_url = f"https://{self.akv_name}.vault.azure.net", +from azure.keyvault.secrets import SecretClient +secret_client = SecretClient( + vault_url = f"https://.vault.azure.net", credential=DefaultAzureCredential() ) +feathr_client = FeathrClient(..., secret_manager_client = secret_client) ``` -For AWS Secrets Manager: +For AWS Secrets Manager, users need to create a SecretCache object and pass it to Feathr client, like below: ```python +import botocore +import botocore.session +from aws_secretsmanager_caching import SecretCache, SecretCacheConfig + client = botocore.session.get_session().create_client( service_name='secretsmanager', aws_access_key_id = '', @@ -73,6 +78,8 @@ client = botocore.session.get_session().create_client( ) cache_config = SecretCacheConfig() cache = SecretCache( config = cache_config, client = client) +feathr_client = FeathrClient(..., secret_manager_client = cache) + ``` # A list of environment variables that Feathr uses From f616522ba6066da7d516b0026e3517012a2cfdfb Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Tue, 27 Sep 2022 07:49:04 -0700 Subject: [PATCH 04/17] Update _envvariableutil.py --- feathr_project/feathr/utils/_envvariableutil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feathr_project/feathr/utils/_envvariableutil.py b/feathr_project/feathr/utils/_envvariableutil.py index 9d2863461..168bcda88 100644 --- a/feathr_project/feathr/utils/_envvariableutil.py +++ b/feathr_project/feathr/utils/_envvariableutil.py @@ -5,7 +5,7 @@ from feathr.secrets.aws_secretmanager import AWSSecretManagerClient class _EnvVaraibleUtil(object): - def __init__(self, config_path: str, secret_manager_client): + def __init__(self, config_path: str, secret_manager_client = None): """Initialize the environment variable utils client Args: @@ -14,7 +14,7 @@ def __init__(self, config_path: str, secret_manager_client): """ self.config_path = config_path # Set to none first to avoid invalid reference - self.secret_manager_client = None + self.secret_manager_client = secret_manager_client if self.get_environment_variable_with_default('secrets', 'azure_key_vault', 'name'): self.secret_manager_client = AzureKeyVaultClient( secret_namespace=self.get_environment_variable_with_default('secrets', 'azure_key_vault', 'name'), From cdcd61294a4fd8b3844ee0fb4e04505ca590ad1f Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Wed, 28 Sep 2022 01:41:15 -0700 Subject: [PATCH 05/17] add tests for aws secrets manager --- feathr_project/feathr/client.py | 30 ++---- feathr_project/feathr/secrets/akv_client.py | 13 ++- .../feathr/secrets/aws_secretmanager.py | 16 +-- feathr_project/test/test_fixture.py | 75 ++++++++++++++ feathr_project/test/test_secrets_read.py | 97 +++++++++---------- ...config_secret_test_aws_secret_manager.yaml | 29 ++++++ ...hr_config_secret_test_azure_key_vault.yaml | 29 ++++++ 7 files changed, 203 insertions(+), 86 deletions(-) create mode 100644 feathr_project/test/test_user_workspace/feathr_config_secret_test_aws_secret_manager.yaml create mode 100644 feathr_project/test/test_user_workspace/feathr_config_secret_test_azure_key_vault.yaml diff --git a/feathr_project/feathr/client.py b/feathr_project/feathr/client.py index 9cf1e7108..3c5504d21 100644 --- a/feathr_project/feathr/client.py +++ b/feathr_project/feathr/client.py @@ -170,17 +170,6 @@ def __init__(self, config_path:str = "./feathr_config.yaml", local_workspace_dir # initialize registry self.registry = default_registry_client(self.project_name, config_path=config_path, credential=self.credential) - def _check_required_environment_variables_exist(self): - """Checks if the required environment variables(form feathr_config.yaml) is set. - - Some required information has to be set via environment variables so the client can work. - """ - props = self.secret_names - for required_field in (self.required_fields + props): - if required_field not in os.environ: - raise RuntimeError(f'{required_field} is not set in environment variable. All required environment ' - f'variables are: {self.required_fields}.') - def register_features(self, from_context: bool = True): """Registers features based on the current workspace @@ -399,13 +388,12 @@ def _construct_redis_client(self): port = self.redis_port ssl_enabled = self.redis_ssl_enabled - redis_client = redis.Redis( + self.redis_client = redis.Redis( host=host, port=port, password=password, ssl=ssl_enabled) self.logger.info('Redis connection is successful and completed.') - self.redis_client = redis_client def get_offline_features(self, @@ -570,7 +558,7 @@ def monitor_features(self, settings: MonitoringSettings, execution_configuration # Should search in both 'derived_feature_list' and 'anchor_list' # Return related keys(key_column list) or None if cannot find the feature def _get_feature_key(self, feature_name: str): - features = [] + features: List[FeatureBase] = [] if 'derived_feature_list' in dir(self): features += self.derived_feature_list if 'anchor_list' in dir(self): @@ -732,7 +720,7 @@ def _get_s3_config_str(self): # keys can't be only accessed through environment access_key = self.envutils.get_environment_variable('S3_ACCESS_KEY') secret_key = self.envutils.get_environment_variable('S3_SECRET_KEY') - # HOCCON format will be parsed by the Feathr job + # HOCON format will be parsed by the Feathr job config_str = """ S3_ENDPOINT: {S3_ENDPOINT} S3_ACCESS_KEY: "{S3_ACCESS_KEY}" @@ -747,7 +735,7 @@ def _get_adls_config_str(self): # if ADLS Account is set in the feathr_config, then we need other environment variables # keys can't be only accessed through environment key = self.envutils.get_environment_variable('ADLS_KEY') - # HOCCON format will be parsed by the Feathr job + # HOCON format will be parsed by the Feathr job config_str = """ ADLS_ACCOUNT: {ADLS_ACCOUNT} ADLS_KEY: "{ADLS_KEY}" @@ -761,7 +749,7 @@ def _get_blob_config_str(self): # if BLOB Account is set in the feathr_config, then we need other environment variables # keys can't be only accessed through environment key = self.envutils.get_environment_variable('BLOB_KEY') - # HOCCON format will be parsed by the Feathr job + # HOCON format will be parsed by the Feathr job config_str = """ BLOB_ACCOUNT: {BLOB_ACCOUNT} BLOB_KEY: "{BLOB_KEY}" @@ -777,7 +765,7 @@ def _get_sql_config_str(self): driver = self.envutils.get_environment_variable('JDBC_DRIVER') auth_flag = self.envutils.get_environment_variable('JDBC_AUTH_FLAG') token = self.envutils.get_environment_variable('JDBC_TOKEN') - # HOCCON format will be parsed by the Feathr job + # HOCON format will be parsed by the Feathr job config_str = """ JDBC_TABLE: {JDBC_TABLE} JDBC_USER: {JDBC_USER} @@ -794,7 +782,7 @@ def _get_monitoring_config_str(self): user = self.envutils.get_environment_variable_with_default('monitoring', 'database', 'sql', 'user') password = self.envutils.get_environment_variable('MONITORING_DATABASE_SQL_PASSWORD') if url: - # HOCCON format will be parsed by the Feathr job + # HOCON format will be parsed by the Feathr job config_str = """ MONITORING_DATABASE_SQL_URL: "{url}" MONITORING_DATABASE_SQL_USER: {user} @@ -811,7 +799,7 @@ def _get_snowflake_config_str(self): sf_user = self.envutils.get_environment_variable_with_default('offline_store', 'snowflake', 'user') sf_role = self.envutils.get_environment_variable_with_default('offline_store', 'snowflake', 'role') sf_password = self.envutils.get_environment_variable('JDBC_SF_PASSWORD') - # HOCCON format will be parsed by the Feathr job + # HOCON format will be parsed by the Feathr job config_str = """ JDBC_SF_URL: {JDBC_SF_URL} JDBC_SF_USER: {JDBC_SF_USER} @@ -824,7 +812,7 @@ def _get_kafka_config_str(self): """Construct the Kafka config string. The endpoint, access key, secret key, and other parameters can be set via environment variables.""" sasl = self.envutils.get_environment_variable('KAFKA_SASL_JAAS_CONFIG') - # HOCCON format will be parsed by the Feathr job + # HOCON format will be parsed by the Feathr job config_str = """ KAFKA_SASL_JAAS_CONFIG: "{sasl}" """.format(sasl=sasl) diff --git a/feathr_project/feathr/secrets/akv_client.py b/feathr_project/feathr/secrets/akv_client.py index 53f2d3e31..d8baf7bcc 100644 --- a/feathr_project/feathr/secrets/akv_client.py +++ b/feathr_project/feathr/secrets/akv_client.py @@ -6,10 +6,12 @@ class AzureKeyVaultClient(FeathrSecretsManagementClient): def __init__(self, secret_namespace: str, secret_client: SecretClient = None): - if isinstance(secret_client, SecretClient): - self.secret_client = secret_client - else: - raise RuntimeError("You need to pass an azure.keyvault.secrets.SecretClient instance.") + """Initializes the AzureKeyVaultClient. Note that `secret_namespace` is not used, since the namespace information will be included in secret_client. + """ + self.secret_client = secret_client + if self.secret_client is not None and not isinstance(secret_client, SecretClient): + raise RuntimeError( + "You need to pass an azure.keyvault.secrets.SecretClient instance.") def get_feathr_secret(self, secret_name: str) -> str: """Get Feathr Secrets from Azure Key Vault. Note that this function will replace '_' in `secret_name` with '-' since Azure Key Vault doesn't support it @@ -17,6 +19,9 @@ def get_feathr_secret(self, secret_name: str) -> str: Returns: str: returned secret from secret management service """ + if self.secret_client is None: + raise RuntimeError("You need to pass an azure.keyvault.secrets.SecretClient instance when initializing FeathrClient.") + try: # replace '_' with '-' since Azure Key Vault doesn't support it variable_replaced = secret_name.replace('_', '-') # .upper() diff --git a/feathr_project/feathr/secrets/aws_secretmanager.py b/feathr_project/feathr/secrets/aws_secretmanager.py index 9271aae34..247b6ad0f 100644 --- a/feathr_project/feathr/secrets/aws_secretmanager.py +++ b/feathr_project/feathr/secrets/aws_secretmanager.py @@ -1,29 +1,29 @@ from loguru import logger import json from feathr.secrets.abc import FeathrSecretsManagementClient -import botocore -from botocore.exceptions import ClientError from aws_secretsmanager_caching.secret_cache import SecretCache class AWSSecretManagerClient(FeathrSecretsManagementClient): def __init__(self, secret_namespace: str = None, secret_client: SecretCache = None): self.secret_id = secret_namespace - self.secret_namespace = secret_namespace - + self.secret_client = secret_client # make sure secret_client is a SecretCache type - if not isinstance(secret_client, SecretCache): + if secret_client is not None and not isinstance(secret_client, SecretCache): raise RuntimeError( "You need to pass a aws_secretsmanager_caching.secret_cache.SecretCache instance. Please refer to https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_cache-python.html for more details.") - else: - self.secret_client = secret_client def get_feathr_secret(self, secret_string: str): """Get Feathr Secrets from AWS Secrets manager. It's also recommended that the client passes a cache objects to reduce cost. See more details here: https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_cache-python.html """ + if self.secret_client is None: + raise RuntimeError( + "You need to pass a aws_secretsmanager_caching.secret_cache.SecretCache instance when initializing FeathrClient.") + try: - get_secret_value_response = self.secret_client.get_secret_string(self.secret_namespace) + get_secret_value_response = self.secret_client.get_secret_string( + self.secret_id) # result is in str format, so we need to load it as a dict secret = json.loads(get_secret_value_response) return secret[secret_string] diff --git a/feathr_project/test/test_fixture.py b/feathr_project/test/test_fixture.py index d13a261ef..de99d7f1a 100644 --- a/feathr_project/test/test_fixture.py +++ b/feathr_project/test/test_fixture.py @@ -44,6 +44,81 @@ def basic_test_setup(config_path: str): ] + request_anchor = FeatureAnchor(name="request_features", + source=INPUT_CONTEXT, + features=features) + + f_trip_time_distance = DerivedFeature(name="f_trip_time_distance", + feature_type=FLOAT, + input_features=[ + f_trip_distance, f_trip_time_duration], + transform="f_trip_distance * f_trip_time_duration") + + f_trip_time_rounded = DerivedFeature(name="f_trip_time_rounded", + feature_type=INT32, + input_features=[f_trip_time_duration], + transform="f_trip_time_duration % 10") + + location_id = TypedKey(key_column="DOLocationID", + key_column_type=ValueType.INT32, + description="location id in NYC", + full_name="nyc_taxi.location_id") + agg_features = [Feature(name="f_location_avg_fare", + key=location_id, + feature_type=FLOAT, + transform=WindowAggTransformation(agg_expr="cast_float(fare_amount)", + agg_func="AVG", + window="90d", + filter="fare_amount > 0" + )), + Feature(name="f_location_max_fare", + key=location_id, + feature_type=FLOAT, + transform=WindowAggTransformation(agg_expr="cast_float(fare_amount)", + agg_func="MAX", + window="90d")) + ] + + agg_anchor = FeatureAnchor(name="aggregationFeatures", + source=batch_source, + features=agg_features) + + client.build_features(anchor_list=[agg_anchor, request_anchor], derived_feature_list=[ + f_trip_time_distance, f_trip_time_rounded]) + + return client + +def secret_test_setup(config_path: str, secret_manager_client): + + now = datetime.now() + # set workspace folder by time; make sure we don't have write conflict if there are many CI tests running + os.environ['SPARK_CONFIG__DATABRICKS__WORK_DIR'] = ''.join(['dbfs:/feathrazure_cijob','_', str(now.minute), '_', str(now.second), '_', str(now.microsecond)]) + os.environ['SPARK_CONFIG__AZURE_SYNAPSE__WORKSPACE_DIR'] = ''.join(['abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_github_ci','_', str(now.minute), '_', str(now.second) ,'_', str(now.microsecond)]) + + client = FeathrClient(config_path=config_path, secret_manager_client=secret_manager_client) + batch_source = HdfsSource(name="nycTaxiBatchSource", + path="wasbs://public@azurefeathrstorage.blob.core.windows.net/sample_data/green_tripdata_2020-04.csv", + event_timestamp_column="lpep_dropoff_datetime", + timestamp_format="yyyy-MM-dd HH:mm:ss") + + f_trip_distance = Feature(name="f_trip_distance", + feature_type=FLOAT, transform="trip_distance") + f_trip_time_duration = Feature(name="f_trip_time_duration", + feature_type=INT32, + transform="(to_unix_timestamp(lpep_dropoff_datetime) - to_unix_timestamp(lpep_pickup_datetime))/60") + + features = [ + f_trip_distance, + f_trip_time_duration, + Feature(name="f_is_long_trip_distance", + feature_type=BOOLEAN, + transform="cast_float(trip_distance)>30"), + Feature(name="f_day_of_week", + feature_type=INT32, + transform="dayofweek(lpep_dropoff_datetime)"), + ] + + request_anchor = FeatureAnchor(name="request_features", source=INPUT_CONTEXT, features=features) diff --git a/feathr_project/test/test_secrets_read.py b/feathr_project/test/test_secrets_read.py index 2e5916825..88777e281 100644 --- a/feathr_project/test/test_secrets_read.py +++ b/feathr_project/test/test_secrets_read.py @@ -1,68 +1,59 @@ import os -from datetime import datetime from pathlib import Path -from unittest import result - -from click.testing import CliRunner -from feathr import (BOOLEAN, FLOAT, INT32, FeatureQuery, ObservationSettings, - SparkExecutionConfiguration, TypedKey, ValueType) +from azure.identity import DefaultAzureCredential from feathr.client import FeathrClient -from feathr.utils.job_utils import get_result_df - -from test_fixture import basic_test_setup +from azure.keyvault.secrets import SecretClient +from test_fixture import secret_test_setup from feathr.constants import OUTPUT_FORMAT +import botocore +import botocore.session +from aws_secretsmanager_caching import SecretCache, SecretCacheConfig -# test parquet file read/write without an extension name -def test_feathr_get_secrets_from_key_vault(): + +def test_feathr_get_secrets_from_azure_key_vault(): """ - Test if the program can read the key vault secrets as expected + Test if the program can read azure key vault secrets as expected """ # TODO: need to test get_environment_variable() as well - os.environ['SECRETS__AZURE_KEY_VAULT__NAME'] = 'feathrazuretest3-kv' + test_workspace_dir = Path( + __file__).parent.resolve() / "test_user_workspace" + + secret_client = SecretClient( + vault_url=f"https://{self.akv_name}.vault.azure.net", + credential=DefaultAzureCredential() + ) + client: FeathrClient = secret_test_setup(os.path.join(test_workspace_dir, "feathr_config_secret_test_azure_key_vault.yaml"), secret_manager_client=secret_client) + + # `redis_host` should be there since it's not available in the environment variable, and not in the config file, we expect we get it from azure key_vault + assert client.redis_host is not None - # the config below doesn't have `ONLINE_STORE__REDIS__HOST` for testing purpose - yaml_config = """ - project_config: - project_name: 'project_feathr_integration_test' - offline_store: - s3: - s3_enabled: true - s3_endpoint: 's3.amazonaws.com' - snowflake: - url: "dqllago-ol19457.snowflakecomputing.com" - user: "feathrintegration" - role: "ACCOUNTADMIN" - spark_config: - spark_cluster: 'databricks' - spark_result_output_parts: '1' - azure_synapse: - dev_url: 'https://feathrazuretest3synapse.dev.azuresynapse.net' - pool_name: 'spark3' - workspace_dir: 'abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace' - executor_size: 'Small' - executor_num: 1 - databricks: - workspace_instance_url: 'https://adb-2474129336842816.16.azuredatabricks.net/' - workspace_token_value: '' - config_template: '{"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}}' - work_dir: 'dbfs:/feathr_getting_started' - feathr_runtime_location: '' - online_store: - redis: - port: 6380 - ssl_enabled: True - feature_registry: - purview: - type_system_initialization: false - purview_name: 'feathrazuretest3-purview1' - delimiter: '__' +# test parquet file read/write without an extension name +def test_feathr_get_secrets_from_aws_secret_manager(): """ + Test if the program can read AWS secret manager as expected + """ + test_workspace_dir = Path( + __file__).parent.resolve() / "test_user_workspace" + + secret_name = "secretname3" + region_name = "us-east-1" - with open("/tmp/feathr_config.yaml", "w") as text_file: - text_file.write(yaml_config) + # Create a Secrets Manager client + # session = boto3.Session() + # s3 = session.client('s3') + # ddb = session.resource('dynamodb') + + client = botocore.session.get_session().create_client( + service_name='secretsmanager', + aws_access_key_id = 'AKIA4C4QVX4ITOHSBR3D', + aws_secret_access_key= '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', + region_name=region_name + ) + cache_config = SecretCacheConfig() + cache = SecretCache( config = cache_config, client = client) + + client: FeathrClient = secret_test_setup(os.path.join(test_workspace_dir, "feathr_config_secret_test_aws_secret_manager.yaml"), secret_manager_client=cache) - client = FeathrClient(config_path="/tmp/feathr_config.yaml") # `redis_host` should be there since it's not available in the environment variable, and not in the config file, we expect we get it from azure key_vault assert client.redis_host is not None - diff --git a/feathr_project/test/test_user_workspace/feathr_config_secret_test_aws_secret_manager.yaml b/feathr_project/test/test_user_workspace/feathr_config_secret_test_aws_secret_manager.yaml new file mode 100644 index 000000000..d08512bb7 --- /dev/null +++ b/feathr_project/test/test_user_workspace/feathr_config_secret_test_aws_secret_manager.yaml @@ -0,0 +1,29 @@ +project_config: + project_name: "project_feathr_integration_test" +offline_store: + s3: + s3_enabled: true + s3_endpoint: "s3.amazonaws.com" +spark_config: + spark_cluster: "databricks" + spark_result_output_parts: "1" + azure_synapse: + dev_url: "https://feathrazuretest3synapse.dev.azuresynapse.net" + pool_name: "spark3" + workspace_dir: "abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace" + executor_size: "Small" + executor_num: 1 + + databricks: + workspace_instance_url: "https://adb-2474129336842816.16.azuredatabricks.net/" + workspace_token_value: "" + config_template: '{"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}}' + work_dir: "dbfs:/feathr_getting_started" + feathr_runtime_location: "" +online_store: + redis: + port: 6380 + ssl_enabled: True +secrets: + aws_secrets_manager: + secret_id: feathrsecretsmanagerci diff --git a/feathr_project/test/test_user_workspace/feathr_config_secret_test_azure_key_vault.yaml b/feathr_project/test/test_user_workspace/feathr_config_secret_test_azure_key_vault.yaml new file mode 100644 index 000000000..d473566ff --- /dev/null +++ b/feathr_project/test/test_user_workspace/feathr_config_secret_test_azure_key_vault.yaml @@ -0,0 +1,29 @@ +project_config: + project_name: "project_feathr_integration_test" +offline_store: + s3: + s3_enabled: true + s3_endpoint: "s3.amazonaws.com" +spark_config: + spark_cluster: "databricks" + spark_result_output_parts: "1" + azure_synapse: + dev_url: "https://feathrazuretest3synapse.dev.azuresynapse.net" + pool_name: "spark3" + workspace_dir: "abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace" + executor_size: "Small" + executor_num: 1 + + databricks: + workspace_instance_url: "https://adb-2474129336842816.16.azuredatabricks.net/" + workspace_token_value: "" + config_template: '{"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}}' + work_dir: "dbfs:/feathr_getting_started" + feathr_runtime_location: "" +online_store: + redis: + port: 6380 + ssl_enabled: True +secrets: + azure_key_vault: + name: feathrazuretest3-kv From aa5fdda01af49729d371efd5dc762ffc1ce00cae Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Wed, 28 Sep 2022 02:02:58 -0700 Subject: [PATCH 06/17] Update test_secrets_read.py --- feathr_project/test/test_secrets_read.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/feathr_project/test/test_secrets_read.py b/feathr_project/test/test_secrets_read.py index 88777e281..985080dac 100644 --- a/feathr_project/test/test_secrets_read.py +++ b/feathr_project/test/test_secrets_read.py @@ -46,8 +46,6 @@ def test_feathr_get_secrets_from_aws_secret_manager(): client = botocore.session.get_session().create_client( service_name='secretsmanager', - aws_access_key_id = 'AKIA4C4QVX4ITOHSBR3D', - aws_secret_access_key= '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', region_name=region_name ) cache_config = SecretCacheConfig() From a6870d948553f0a9bfe9afcecdf741ff08cd7f64 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Wed, 28 Sep 2022 22:04:27 -0700 Subject: [PATCH 07/17] fix tests --- feathr_project/feathr/secrets/abc.py | 2 +- .../feathr/secrets/aws_secretmanager.py | 6 ++-- .../feathr/utils/_envvariableutil.py | 10 +++---- feathr_project/test/test_secrets_read.py | 4 +-- ...config_secret_test_aws_secret_manager.yaml | 30 +++++++++---------- ...hr_config_secret_test_azure_key_vault.yaml | 30 +++++++++---------- 6 files changed, 39 insertions(+), 43 deletions(-) diff --git a/feathr_project/feathr/secrets/abc.py b/feathr_project/feathr/secrets/abc.py index 7ae20ebe1..fae681265 100644 --- a/feathr_project/feathr/secrets/abc.py +++ b/feathr_project/feathr/secrets/abc.py @@ -21,7 +21,7 @@ def __init__(self, secret_namespace: str, secret_client) -> None: pass @abstractmethod - def get_feathr_secret(self, secret_string: str) -> str: + def get_feathr_secret(self, secret_name: str) -> str: """Get Feathr Secrets from a certain secret management service, such as Azure Key Vault or AWS Secrets Manager. Returns: diff --git a/feathr_project/feathr/secrets/aws_secretmanager.py b/feathr_project/feathr/secrets/aws_secretmanager.py index 247b6ad0f..feb6586f9 100644 --- a/feathr_project/feathr/secrets/aws_secretmanager.py +++ b/feathr_project/feathr/secrets/aws_secretmanager.py @@ -13,7 +13,7 @@ def __init__(self, secret_namespace: str = None, secret_client: SecretCache = No raise RuntimeError( "You need to pass a aws_secretsmanager_caching.secret_cache.SecretCache instance. Please refer to https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_cache-python.html for more details.") - def get_feathr_secret(self, secret_string: str): + def get_feathr_secret(self, secret_name: str): """Get Feathr Secrets from AWS Secrets manager. It's also recommended that the client passes a cache objects to reduce cost. See more details here: https://docs.aws.amazon.com/secretsmanager/latest/userguide/retrieving-secrets_cache-python.html """ @@ -26,8 +26,8 @@ def get_feathr_secret(self, secret_string: str): self.secret_id) # result is in str format, so we need to load it as a dict secret = json.loads(get_secret_value_response) - return secret[secret_string] + return secret[secret_name] except KeyError as e: logger.error( - f"Secret {secret_string} cannot be found in secretsmanager {self.secret_id}.") + f"Secret {secret_name} cannot be found in secretsmanager {self.secret_id}.") raise e diff --git a/feathr_project/feathr/utils/_envvariableutil.py b/feathr_project/feathr/utils/_envvariableutil.py index 168bcda88..eb05d7880 100644 --- a/feathr_project/feathr/utils/_envvariableutil.py +++ b/feathr_project/feathr/utils/_envvariableutil.py @@ -14,15 +14,15 @@ def __init__(self, config_path: str, secret_manager_client = None): """ self.config_path = config_path # Set to none first to avoid invalid reference - self.secret_manager_client = secret_manager_client - if self.get_environment_variable_with_default('secrets', 'azure_key_vault', 'name'): + self.secret_manager_client = None + if secret_manager_client and self.get_environment_variable_with_default('secrets', 'azure_key_vault', 'name'): self.secret_manager_client = AzureKeyVaultClient( secret_namespace=self.get_environment_variable_with_default('secrets', 'azure_key_vault', 'name'), - secret_client=self.secret_manager_client) - elif self.get_environment_variable_with_default('secrets', 'aws_secrets_manager', 'secret_id'): + secret_client=secret_manager_client) + elif secret_manager_client and self.get_environment_variable_with_default('secrets', 'aws_secrets_manager', 'secret_id'): self.secret_manager_client = AWSSecretManagerClient( secret_namespace=self.get_environment_variable_with_default('secrets', 'aws_secrets_manager', 'secret_id'), - secret_client=self.secret_manager_client) + secret_client=secret_manager_client) def get_environment_variable_with_default(self, *args): """Gets the environment variable for the variable key. diff --git a/feathr_project/test/test_secrets_read.py b/feathr_project/test/test_secrets_read.py index 985080dac..a2b32f07b 100644 --- a/feathr_project/test/test_secrets_read.py +++ b/feathr_project/test/test_secrets_read.py @@ -19,8 +19,8 @@ def test_feathr_get_secrets_from_azure_key_vault(): __file__).parent.resolve() / "test_user_workspace" secret_client = SecretClient( - vault_url=f"https://{self.akv_name}.vault.azure.net", - credential=DefaultAzureCredential() + vault_url="https://feathrazuretest3-kv.vault.azure.net", # hard code the CI key vault endpoint + credential=DefaultAzureCredential(exclude_cli_credential=False,exclude_interactive_browser_credential=False) ) client: FeathrClient = secret_test_setup(os.path.join(test_workspace_dir, "feathr_config_secret_test_azure_key_vault.yaml"), secret_manager_client=secret_client) diff --git a/feathr_project/test/test_user_workspace/feathr_config_secret_test_aws_secret_manager.yaml b/feathr_project/test/test_user_workspace/feathr_config_secret_test_aws_secret_manager.yaml index d08512bb7..af119abf1 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_secret_test_aws_secret_manager.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_secret_test_aws_secret_manager.yaml @@ -2,24 +2,22 @@ project_config: project_name: "project_feathr_integration_test" offline_store: s3: - s3_enabled: true - s3_endpoint: "s3.amazonaws.com" + s3_enabled: false + adls: + adls_enabled: false + wasb: + wasb_enabled: false + jdbc: + jdbc_enabled: false + snowflake: + snowflake_enabled: false spark_config: - spark_cluster: "databricks" + spark_cluster: "local" spark_result_output_parts: "1" - azure_synapse: - dev_url: "https://feathrazuretest3synapse.dev.azuresynapse.net" - pool_name: "spark3" - workspace_dir: "abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace" - executor_size: "Small" - executor_num: 1 - - databricks: - workspace_instance_url: "https://adb-2474129336842816.16.azuredatabricks.net/" - workspace_token_value: "" - config_template: '{"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}}' - work_dir: "dbfs:/feathr_getting_started" - feathr_runtime_location: "" + local: + feathr_runtime_location: None + workspace: None + master: None online_store: redis: port: 6380 diff --git a/feathr_project/test/test_user_workspace/feathr_config_secret_test_azure_key_vault.yaml b/feathr_project/test/test_user_workspace/feathr_config_secret_test_azure_key_vault.yaml index d473566ff..b2ee87976 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_secret_test_azure_key_vault.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_secret_test_azure_key_vault.yaml @@ -2,24 +2,22 @@ project_config: project_name: "project_feathr_integration_test" offline_store: s3: - s3_enabled: true - s3_endpoint: "s3.amazonaws.com" + s3_enabled: false + adls: + adls_enabled: false + wasb: + wasb_enabled: false + jdbc: + jdbc_enabled: false + snowflake: + snowflake_enabled: false spark_config: - spark_cluster: "databricks" + spark_cluster: "local" spark_result_output_parts: "1" - azure_synapse: - dev_url: "https://feathrazuretest3synapse.dev.azuresynapse.net" - pool_name: "spark3" - workspace_dir: "abfss://feathrazuretest3fs@feathrazuretest3storage.dfs.core.windows.net/feathr_test_workspace" - executor_size: "Small" - executor_num: 1 - - databricks: - workspace_instance_url: "https://adb-2474129336842816.16.azuredatabricks.net/" - workspace_token_value: "" - config_template: '{"run_name":"FEATHR_FILL_IN","new_cluster":{"spark_version":"9.1.x-scala2.12","num_workers":1,"spark_conf":{"FEATHR_FILL_IN":"FEATHR_FILL_IN"},"instance_pool_id":"0403-214809-inlet434-pool-l9dj3kwz"},"libraries":[{"jar":"FEATHR_FILL_IN"}],"spark_jar_task":{"main_class_name":"FEATHR_FILL_IN","parameters":["FEATHR_FILL_IN"]}}' - work_dir: "dbfs:/feathr_getting_started" - feathr_runtime_location: "" + local: + feathr_runtime_location: None + workspace: None + master: None online_store: redis: port: 6380 From 997a2b149fc2ac28e615c21d550e50690ee2d059 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Wed, 28 Sep 2022 22:15:08 -0700 Subject: [PATCH 08/17] Update test_secrets_read.py --- feathr_project/test/test_secrets_read.py | 34 +++++++++++------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/feathr_project/test/test_secrets_read.py b/feathr_project/test/test_secrets_read.py index a2b32f07b..e61d3f667 100644 --- a/feathr_project/test/test_secrets_read.py +++ b/feathr_project/test/test_secrets_read.py @@ -5,9 +5,9 @@ from azure.keyvault.secrets import SecretClient from test_fixture import secret_test_setup from feathr.constants import OUTPUT_FORMAT -import botocore -import botocore.session -from aws_secretsmanager_caching import SecretCache, SecretCacheConfig +import botocore +import botocore.session +from aws_secretsmanager_caching import SecretCache, SecretCacheConfig def test_feathr_get_secrets_from_azure_key_vault(): @@ -19,12 +19,15 @@ def test_feathr_get_secrets_from_azure_key_vault(): __file__).parent.resolve() / "test_user_workspace" secret_client = SecretClient( - vault_url="https://feathrazuretest3-kv.vault.azure.net", # hard code the CI key vault endpoint - credential=DefaultAzureCredential(exclude_cli_credential=False,exclude_interactive_browser_credential=False) + # hard code the CI key vault endpoint + vault_url="https://feathrazuretest3-kv.vault.azure.net", + credential=DefaultAzureCredential( + exclude_cli_credential=False, exclude_interactive_browser_credential=False) ) - client: FeathrClient = secret_test_setup(os.path.join(test_workspace_dir, "feathr_config_secret_test_azure_key_vault.yaml"), secret_manager_client=secret_client) + client: FeathrClient = secret_test_setup(os.path.join( + test_workspace_dir, "feathr_config_secret_test_azure_key_vault.yaml"), secret_manager_client=secret_client) - # `redis_host` should be there since it's not available in the environment variable, and not in the config file, we expect we get it from azure key_vault + # `redis_host` should be read from secret management service since it's not available in the environment variable, and not in the config file, we expect we get it from azure key_vault assert client.redis_host is not None @@ -36,22 +39,15 @@ def test_feathr_get_secrets_from_aws_secret_manager(): test_workspace_dir = Path( __file__).parent.resolve() / "test_user_workspace" - secret_name = "secretname3" - region_name = "us-east-1" - - # Create a Secrets Manager client - # session = boto3.Session() - # s3 = session.client('s3') - # ddb = session.resource('dynamodb') - client = botocore.session.get_session().create_client( service_name='secretsmanager', - region_name=region_name + region_name="us-east-1" ) cache_config = SecretCacheConfig() - cache = SecretCache( config = cache_config, client = client) + secret_cache = SecretCache(config=cache_config, client=client) - client: FeathrClient = secret_test_setup(os.path.join(test_workspace_dir, "feathr_config_secret_test_aws_secret_manager.yaml"), secret_manager_client=cache) + client: FeathrClient = secret_test_setup(os.path.join( + test_workspace_dir, "feathr_config_secret_test_aws_secret_manager.yaml"), secret_manager_client=secret_cache) - # `redis_host` should be there since it's not available in the environment variable, and not in the config file, we expect we get it from azure key_vault + # `redis_host` should be read from secret management service since it's not available in the environment variable, and not in the config file, we expect we get it from azure key_vault assert client.redis_host is not None From 8be6a424e6bb47f554ffa229074c37f781f55b39 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 29 Sep 2022 00:49:35 -0700 Subject: [PATCH 09/17] fix test --- .github/workflows/pull_request_push_test.yml | 5 ++++- .../feathr_config_secret_test_aws_secret_manager.yaml | 3 +++ .../feathr_config_secret_test_azure_key_vault.yaml | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull_request_push_test.yml b/.github/workflows/pull_request_push_test.yml index 15981ffc4..49dde7e40 100644 --- a/.github/workflows/pull_request_push_test.yml +++ b/.github/workflows/pull_request_push_test.yml @@ -123,7 +123,8 @@ jobs: COSMOS1_KEY: ${{secrets.COSMOS1_KEY}} SQL1_USER: ${{secrets.SQL1_USER}} SQL1_PASSWORD: ${{secrets.SQL1_PASSWORD}} - + AWS_ACCESS_KEY_ID: ${{AWS_ACCESS_KEY_ID}} + AWS_SECRET_ACCESS_KEY: ${{AWS_SECRET_ACCESS_KEY}} run: | # run only test with databricks. run in 4 parallel jobs pytest -n 6 feathr_project/test/ @@ -193,6 +194,8 @@ jobs: COSMOS1_KEY: ${{secrets.COSMOS1_KEY}} SQL1_USER: ${{secrets.SQL1_USER}} SQL1_PASSWORD: ${{secrets.SQL1_PASSWORD}} + AWS_ACCESS_KEY_ID: ${{AWS_ACCESS_KEY_ID}} + AWS_SECRET_ACCESS_KEY: ${{AWS_SECRET_ACCESS_KEY}} run: | # skip databricks related test as we just ran the test; also seperate databricks and synapse test to make sure there's no write conflict # run in 4 parallel jobs to make the time shorter diff --git a/feathr_project/test/test_user_workspace/feathr_config_secret_test_aws_secret_manager.yaml b/feathr_project/test/test_user_workspace/feathr_config_secret_test_aws_secret_manager.yaml index af119abf1..4251f7d36 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_secret_test_aws_secret_manager.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_secret_test_aws_secret_manager.yaml @@ -25,3 +25,6 @@ online_store: secrets: aws_secrets_manager: secret_id: feathrsecretsmanagerci +feature_registry: + # The API endpoint of the registry service + api_endpoint: "https://feathr-sql-registry.azurewebsites.net/api/v1" diff --git a/feathr_project/test/test_user_workspace/feathr_config_secret_test_azure_key_vault.yaml b/feathr_project/test/test_user_workspace/feathr_config_secret_test_azure_key_vault.yaml index b2ee87976..a787edd90 100644 --- a/feathr_project/test/test_user_workspace/feathr_config_secret_test_azure_key_vault.yaml +++ b/feathr_project/test/test_user_workspace/feathr_config_secret_test_azure_key_vault.yaml @@ -25,3 +25,6 @@ online_store: secrets: azure_key_vault: name: feathrazuretest3-kv +feature_registry: + # The API endpoint of the registry service + api_endpoint: "https://feathr-sql-registry.azurewebsites.net/api/v1" From e617b992829de46aefad7b9ec640d99c65f167eb Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Thu, 29 Sep 2022 04:23:52 -0700 Subject: [PATCH 10/17] Update pull_request_push_test.yml --- .github/workflows/pull_request_push_test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request_push_test.yml b/.github/workflows/pull_request_push_test.yml index 49dde7e40..1102d6028 100644 --- a/.github/workflows/pull_request_push_test.yml +++ b/.github/workflows/pull_request_push_test.yml @@ -123,8 +123,8 @@ jobs: COSMOS1_KEY: ${{secrets.COSMOS1_KEY}} SQL1_USER: ${{secrets.SQL1_USER}} SQL1_PASSWORD: ${{secrets.SQL1_PASSWORD}} - AWS_ACCESS_KEY_ID: ${{AWS_ACCESS_KEY_ID}} - AWS_SECRET_ACCESS_KEY: ${{AWS_SECRET_ACCESS_KEY}} + AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} + AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} run: | # run only test with databricks. run in 4 parallel jobs pytest -n 6 feathr_project/test/ @@ -194,8 +194,8 @@ jobs: COSMOS1_KEY: ${{secrets.COSMOS1_KEY}} SQL1_USER: ${{secrets.SQL1_USER}} SQL1_PASSWORD: ${{secrets.SQL1_PASSWORD}} - AWS_ACCESS_KEY_ID: ${{AWS_ACCESS_KEY_ID}} - AWS_SECRET_ACCESS_KEY: ${{AWS_SECRET_ACCESS_KEY}} + AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}} + AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}} run: | # skip databricks related test as we just ran the test; also seperate databricks and synapse test to make sure there's no write conflict # run in 4 parallel jobs to make the time shorter From 9cb332cc34d4aef3b79afa7fb349d1a8ac199712 Mon Sep 17 00:00:00 2001 From: aabbasi-hbo <92401544+aabbasi-hbo@users.noreply.github.com> Date: Fri, 7 Oct 2022 11:07:08 -0700 Subject: [PATCH 11/17] get_secrets_update --- .../feathr/utils/_envvariableutil.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/feathr_project/feathr/utils/_envvariableutil.py b/feathr_project/feathr/utils/_envvariableutil.py index eb05d7880..3415beab0 100644 --- a/feathr_project/feathr/utils/_envvariableutil.py +++ b/feathr_project/feathr/utils/_envvariableutil.py @@ -1,6 +1,7 @@ import os import yaml from loguru import logger +from azure.core.exceptions import ResouceNotFoundError from feathr.secrets.akv_client import AzureKeyVaultClient from feathr.secrets.aws_secretmanager import AWSSecretManagerClient @@ -67,7 +68,16 @@ def get_environment_variable_with_default(self, *args): # If it's not available in the feathr_config.yaml file, Feathr will try to retrieve the value from key vault if self.secret_manager_client: - return self.secret_manager_client.get_feathr_secret(env_keyword) + try: + return self.secret_manager_client.get_feathr_secret(env_keyword) + except ResouceNotFoundError: + # print out warning message if cannot find the env variable in all the resources + logger.warning('Environment variable {} not found in environment variable, default YAML config file, or key vault service.', env_keyword) + return None + except KeyError: + # print out warning message if cannot find the env variable in all the resources + logger.warning('Environment variable {} not found in environment variable, default YAML config file, or key vault service.', env_keyword) + return None def get_environment_variable(self, variable_key): """Gets the environment variable for the variable key. @@ -89,4 +99,13 @@ def get_environment_variable(self, variable_key): logger.info(variable_key + ' is not set in the environment variables.') if self.secret_manager_client: - return self.secret_manager_client.get_feathr_secret(variable_key) + try: + return self.secret_manager_client.get_feathr_secret(variable_key) + except ResouceNotFoundError: + # print out warning message if cannot find the env variable in all the resources + logger.warning('Environment variable {} not found in environment variable, default YAML config file, or key vault service.', variable_key) + return None + except KeyError: + # print out warning message if cannot find the env variable in all the resources + logger.warning('Environment variable {} not found in environment variable, default YAML config file, or key vault service.', variable_key) + return None From 218123f4521df8a6982f37b2501c26ac52c9ea6d Mon Sep 17 00:00:00 2001 From: aabbasi-hbo <92401544+aabbasi-hbo@users.noreply.github.com> Date: Fri, 7 Oct 2022 14:44:25 -0700 Subject: [PATCH 12/17] move import statement --- feathr_project/feathr/utils/_envvariableutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feathr_project/feathr/utils/_envvariableutil.py b/feathr_project/feathr/utils/_envvariableutil.py index 3415beab0..d3c343db0 100644 --- a/feathr_project/feathr/utils/_envvariableutil.py +++ b/feathr_project/feathr/utils/_envvariableutil.py @@ -1,8 +1,8 @@ import os import yaml from loguru import logger -from azure.core.exceptions import ResouceNotFoundError from feathr.secrets.akv_client import AzureKeyVaultClient +from azure.core.exceptions import ResouceNotFoundError from feathr.secrets.aws_secretmanager import AWSSecretManagerClient class _EnvVaraibleUtil(object): From 07a8cf0c857a2224dabdc9f60c8dd133b49ed3e1 Mon Sep 17 00:00:00 2001 From: aabbasi-hbo <92401544+aabbasi-hbo@users.noreply.github.com> Date: Fri, 7 Oct 2022 14:56:31 -0700 Subject: [PATCH 13/17] update spelling --- feathr_project/feathr/utils/_envvariableutil.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/feathr_project/feathr/utils/_envvariableutil.py b/feathr_project/feathr/utils/_envvariableutil.py index d3c343db0..9bd610d37 100644 --- a/feathr_project/feathr/utils/_envvariableutil.py +++ b/feathr_project/feathr/utils/_envvariableutil.py @@ -2,7 +2,7 @@ import yaml from loguru import logger from feathr.secrets.akv_client import AzureKeyVaultClient -from azure.core.exceptions import ResouceNotFoundError +from azure.core.exceptions import ResourceNotFoundError from feathr.secrets.aws_secretmanager import AWSSecretManagerClient class _EnvVaraibleUtil(object): @@ -70,7 +70,7 @@ def get_environment_variable_with_default(self, *args): if self.secret_manager_client: try: return self.secret_manager_client.get_feathr_secret(env_keyword) - except ResouceNotFoundError: + except ResourceNotFoundError: # print out warning message if cannot find the env variable in all the resources logger.warning('Environment variable {} not found in environment variable, default YAML config file, or key vault service.', env_keyword) return None @@ -101,7 +101,7 @@ def get_environment_variable(self, variable_key): if self.secret_manager_client: try: return self.secret_manager_client.get_feathr_secret(variable_key) - except ResouceNotFoundError: + except ResourceNotFoundError: # print out warning message if cannot find the env variable in all the resources logger.warning('Environment variable {} not found in environment variable, default YAML config file, or key vault service.', variable_key) return None From 44a3ce063f420e8a97cc6669a9cbfaeab17ed88d Mon Sep 17 00:00:00 2001 From: aabbasi-hbo <92401544+aabbasi-hbo@users.noreply.github.com> Date: Fri, 7 Oct 2022 14:59:01 -0700 Subject: [PATCH 14/17] update raise exception --- feathr_project/feathr/secrets/aws_secretmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feathr_project/feathr/secrets/aws_secretmanager.py b/feathr_project/feathr/secrets/aws_secretmanager.py index feb6586f9..105b43336 100644 --- a/feathr_project/feathr/secrets/aws_secretmanager.py +++ b/feathr_project/feathr/secrets/aws_secretmanager.py @@ -30,4 +30,4 @@ def get_feathr_secret(self, secret_name: str): except KeyError as e: logger.error( f"Secret {secret_name} cannot be found in secretsmanager {self.secret_id}.") - raise e + raise From 87cd0838d4b41af6c3cbcc9f6959577f5d49807e Mon Sep 17 00:00:00 2001 From: aabbasi-hbo <92401544+aabbasi-hbo@users.noreply.github.com> Date: Fri, 7 Oct 2022 14:59:37 -0700 Subject: [PATCH 15/17] revert --- feathr_project/feathr/secrets/aws_secretmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feathr_project/feathr/secrets/aws_secretmanager.py b/feathr_project/feathr/secrets/aws_secretmanager.py index 105b43336..feb6586f9 100644 --- a/feathr_project/feathr/secrets/aws_secretmanager.py +++ b/feathr_project/feathr/secrets/aws_secretmanager.py @@ -30,4 +30,4 @@ def get_feathr_secret(self, secret_name: str): except KeyError as e: logger.error( f"Secret {secret_name} cannot be found in secretsmanager {self.secret_id}.") - raise + raise e From 6c2a415cb4ff34c55e099fc3633b52496d1a59c9 Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Tue, 17 Jan 2023 00:23:03 -0800 Subject: [PATCH 16/17] udpate branch --- .../linkedin/feathr/compute/AbstractNode.java | 415 +++++ .../linkedin/feathr/compute/Aggregation.java | 1016 +++++++++++ .../feathr/compute/AggregationFunction.java | 394 ++++ .../feathr/compute/AggregationType.java | 78 + .../com/linkedin/feathr/compute/AnyNode.java | 357 ++++ .../linkedin/feathr/compute/AnyNodeArray.java | 117 ++ .../linkedin/feathr/compute/ComputeGraph.java | 464 +++++ .../linkedin/feathr/compute/ConcreteKey.java | 300 ++++ .../linkedin/feathr/compute/DataSource.java | 1232 +++++++++++++ .../feathr/compute/DataSourceType.java | 43 + .../feathr/compute/DateTimeInterval.java | 377 ++++ .../linkedin/feathr/compute/Dimension.java | 390 ++++ .../feathr/compute/DimensionArray.java | 118 ++ .../feathr/compute/DimensionType.java | 48 + .../com/linkedin/feathr/compute/External.java | 557 ++++++ .../linkedin/feathr/compute/FeatureValue.java | 431 +++++ .../feathr/compute/FeatureVersion.java | 526 ++++++ .../feathr/compute/FrameFeatureType.java | 72 + .../feathr/compute/KeyExpressionType.java | 44 + .../linkedin/feathr/compute/KeyReference.java | 272 +++ .../feathr/compute/KeyReferenceArray.java | 118 ++ .../linkedin/feathr/compute/LateralView.java | 553 ++++++ .../feathr/compute/LateralViewArray.java | 118 ++ .../com/linkedin/feathr/compute/Lookup.java | 1587 +++++++++++++++++ .../feathr/compute/MvelExpression.java | 260 +++ .../com/linkedin/feathr/compute/NodeId.java | 30 + .../feathr/compute/NodeReference.java | 488 +++++ .../feathr/compute/NodeReferenceArray.java | 118 ++ .../feathr/compute/OfflineKeyFunction.java | 477 +++++ .../linkedin/feathr/compute/OperatorId.java | 30 + .../feathr/compute/SlidingWindowFeature.java | 1574 ++++++++++++++++ .../feathr/compute/SqlExpression.java | 260 +++ .../feathr/compute/TensorCategory.java | 42 + .../feathr/compute/TensorFeatureFormat.java | 603 +++++++ .../com/linkedin/feathr/compute/Time.java | 30 + .../linkedin/feathr/compute/TimestampCol.java | 401 +++++ .../feathr/compute/Transformation.java | 1065 +++++++++++ .../compute/TransformationFunction.java | 384 ++++ .../com/linkedin/feathr/compute/Unit.java | 48 + .../feathr/compute/UserDefinedFunction.java | 407 +++++ .../linkedin/feathr/compute/ValueType.java | 66 + .../com/linkedin/feathr/compute/Window.java | 412 +++++ .../feathr/config/join/AbsoluteDateRange.java | 432 +++++ .../feathr/config/join/AbsoluteTimeRange.java | 802 +++++++++ .../com/linkedin/feathr/config/join/Date.java | 575 ++++++ .../config/join/FrameFeatureJoinConfig.java | 520 ++++++ .../linkedin/feathr/config/join/HourTime.java | 727 ++++++++ .../config/join/InputDataTimeSettings.java | 502 ++++++ .../feathr/config/join/JoinTimeSettings.java | 231 +++ .../feathr/config/join/JoiningFeature.java | 1136 ++++++++++++ .../config/join/JoiningFeatureArray.java | 118 ++ .../feathr/config/join/RelativeDateRange.java | 443 +++++ .../feathr/config/join/RelativeTimeRange.java | 453 +++++ .../linkedin/feathr/config/join/Settings.java | 400 +++++ .../config/join/SparkSqlExpression.java | 260 +++ .../feathr/config/join/TimeFormat.java | 31 + .../feathr/config/join/TimeOffset.java | 414 +++++ .../linkedin/feathr/config/join/TimeUnit.java | 48 + .../feathr/config/join/TimeWindow.java | 412 +++++ .../join/TimestampColJoinTimeSettings.java | 432 +++++ .../feathr/config/join/TimestampColumn.java | 609 +++++++ .../join/UseLatestJoinTimeSettings.java | 280 +++ 62 files changed, 25147 insertions(+) create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AbstractNode.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Aggregation.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationFunction.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationType.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNode.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNodeArray.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ComputeGraph.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ConcreteKey.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSource.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSourceType.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DateTimeInterval.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Dimension.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionArray.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionType.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/External.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureValue.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureVersion.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FrameFeatureType.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyExpressionType.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReference.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReferenceArray.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralView.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralViewArray.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Lookup.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/MvelExpression.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeId.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReference.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReferenceArray.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OfflineKeyFunction.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OperatorId.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SlidingWindowFeature.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SqlExpression.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorCategory.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorFeatureFormat.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Time.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TimestampCol.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Transformation.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TransformationFunction.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Unit.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/UserDefinedFunction.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ValueType.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Window.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteDateRange.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteTimeRange.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Date.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/FrameFeatureJoinConfig.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/HourTime.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/InputDataTimeSettings.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoinTimeSettings.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeature.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeatureArray.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeDateRange.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeTimeRange.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Settings.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/SparkSqlExpression.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeFormat.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeOffset.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeUnit.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeWindow.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColJoinTimeSettings.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColumn.java create mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/UseLatestJoinTimeSettings.java diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AbstractNode.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AbstractNode.java new file mode 100644 index 000000000..00f947067 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AbstractNode.java @@ -0,0 +1,415 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * Generic abstraction of a node. All other nodes should derive from this node. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\AbstractNode.pdl.") +public class AbstractNode + extends RecordTemplate +{ + + private final static AbstractNode.Fields _fields = new AbstractNode.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}", SchemaFormatType.PDL)); + private Integer _idField = null; + private ConcreteKey _concreteKeyField = null; + private AbstractNode.ChangeListener __changeListener = new AbstractNode.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); + private final static RecordDataSchema.Field FIELD_ConcreteKey = SCHEMA.getField("concreteKey"); + + public AbstractNode() { + super(new DataMap(3, 0.75F), SCHEMA, 2); + addChangeListener(__changeListener); + } + + public AbstractNode(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static AbstractNode.Fields fields() { + return _fields; + } + + public static AbstractNode.ProjectionMask createMask() { + return new AbstractNode.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for id + * + * @see AbstractNode.Fields#id + */ + public boolean hasId() { + if (_idField!= null) { + return true; + } + return super._map.containsKey("id"); + } + + /** + * Remover for id + * + * @see AbstractNode.Fields#id + */ + public void removeId() { + super._map.remove("id"); + } + + /** + * Getter for id + * + * @see AbstractNode.Fields#id + */ + public Integer getId(GetMode mode) { + switch (mode) { + case STRICT: + return getId(); + case DEFAULT: + case NULL: + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for id + * + * @return + * Required field. Could be null for partial record. + * @see AbstractNode.Fields#id + */ + @Nonnull + public Integer getId() { + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("id"); + } + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + + /** + * Setter for id + * + * @see AbstractNode.Fields#id + */ + public AbstractNode setId(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setId(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.AbstractNode"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeId(); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + } + return this; + } + + /** + * Setter for id + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see AbstractNode.Fields#id + */ + public AbstractNode setId( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.AbstractNode to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + return this; + } + + /** + * Setter for id + * + * @see AbstractNode.Fields#id + */ + public AbstractNode setId(int value) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + return this; + } + + /** + * Existence checker for concreteKey + * + * @see AbstractNode.Fields#concreteKey + */ + public boolean hasConcreteKey() { + if (_concreteKeyField!= null) { + return true; + } + return super._map.containsKey("concreteKey"); + } + + /** + * Remover for concreteKey + * + * @see AbstractNode.Fields#concreteKey + */ + public void removeConcreteKey() { + super._map.remove("concreteKey"); + } + + /** + * Getter for concreteKey + * + * @see AbstractNode.Fields#concreteKey + */ + public ConcreteKey getConcreteKey(GetMode mode) { + return getConcreteKey(); + } + + /** + * Getter for concreteKey + * + * @return + * Optional field. Always check for null. + * @see AbstractNode.Fields#concreteKey + */ + @Nullable + public ConcreteKey getConcreteKey() { + if (_concreteKeyField!= null) { + return _concreteKeyField; + } else { + Object __rawValue = super._map.get("concreteKey"); + _concreteKeyField = ((__rawValue == null)?null:new ConcreteKey(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _concreteKeyField; + } + } + + /** + * Setter for concreteKey + * + * @see AbstractNode.Fields#concreteKey + */ + public AbstractNode setConcreteKey(ConcreteKey value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setConcreteKey(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeConcreteKey(); + } else { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + break; + } + return this; + } + + /** + * Setter for concreteKey + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see AbstractNode.Fields#concreteKey + */ + public AbstractNode setConcreteKey( + @Nonnull + ConcreteKey value) { + if (value == null) { + throw new NullPointerException("Cannot set field concreteKey of com.linkedin.feathr.compute.AbstractNode to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + return this; + } + + @Override + public AbstractNode clone() + throws CloneNotSupportedException + { + AbstractNode __clone = ((AbstractNode) super.clone()); + __clone.__changeListener = new AbstractNode.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public AbstractNode copy() + throws CloneNotSupportedException + { + AbstractNode __copy = ((AbstractNode) super.copy()); + __copy._idField = null; + __copy._concreteKeyField = null; + __copy.__changeListener = new AbstractNode.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final AbstractNode __objectRef; + + private ChangeListener(AbstractNode reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "id": + __objectRef._idField = null; + break; + case "concreteKey": + __objectRef._concreteKeyField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The node would be represented by this id. + * + */ + public PathSpec id() { + return new PathSpec(getPathComponents(), "id"); + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public com.linkedin.feathr.compute.ConcreteKey.Fields concreteKey() { + return new com.linkedin.feathr.compute.ConcreteKey.Fields(getPathComponents(), "concreteKey"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.ConcreteKey.ProjectionMask _concreteKeyMask; + + ProjectionMask() { + super(3); + } + + /** + * The node would be represented by this id. + * + */ + public AbstractNode.ProjectionMask withId() { + getDataMap().put("id", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public AbstractNode.ProjectionMask withConcreteKey(Function nestedMask) { + _concreteKeyMask = nestedMask.apply(((_concreteKeyMask == null)?ConcreteKey.createMask():_concreteKeyMask)); + getDataMap().put("concreteKey", _concreteKeyMask.getDataMap()); + return this; + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public AbstractNode.ProjectionMask withConcreteKey() { + _concreteKeyMask = null; + getDataMap().put("concreteKey", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Aggregation.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Aggregation.java new file mode 100644 index 000000000..934ca5be8 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Aggregation.java @@ -0,0 +1,1016 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * A node to represent an aggregation step. The aggregation inputs like the groupBy field, agg function are delegated to [[AggregationFunction]]. + * This node can represent a feature. As of now, in this step we will be using the SWA library from Spark-algorithms. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Aggregation.pdl.") +public class Aggregation + extends RecordTemplate +{ + + private final static Aggregation.Fields _fields = new Aggregation.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**A node to represent an aggregation step. The aggregation inputs like the groupBy field, agg function are delegated to [[AggregationFunction]].\r\nThis node can represent a feature. As of now, in this step we will be using the SWA library from Spark-algorithms.*/record Aggregation includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node.*/input:/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}/**All the aggregation related parameters and functions are bundled into this.*/function:/**All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node.*/record AggregationFunction{/**The aggregation function.*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**All the aggregation parameters should be bundled into this map. For now, the possible parameters are:-\r\na. target_column - Aggregation column\r\nb. window_size - aggregation window size\r\nc. window unit - aggregation window unit (ex - day, hour)\r\nd. lateral_view_expression - definition of a lateral view for the feature.\r\ne. lateral_view_table_alias - An alias for the lateral view\r\nf. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression.\r\ng. groupBy - groupBy columns. Should be a sparkSql expression.*/parameters:optional map[string,string]}/**If the node is representing a feature, the feature name should be associated with the node.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}", SchemaFormatType.PDL)); + private Integer _idField = null; + private ConcreteKey _concreteKeyField = null; + private NodeReference _inputField = null; + private AggregationFunction _functionField = null; + private String _featureNameField = null; + private FeatureVersion _featureVersionField = null; + private Aggregation.ChangeListener __changeListener = new Aggregation.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); + private final static RecordDataSchema.Field FIELD_ConcreteKey = SCHEMA.getField("concreteKey"); + private final static RecordDataSchema.Field FIELD_Input = SCHEMA.getField("input"); + private final static RecordDataSchema.Field FIELD_Function = SCHEMA.getField("function"); + private final static RecordDataSchema.Field FIELD_FeatureName = SCHEMA.getField("featureName"); + private final static RecordDataSchema.Field FIELD_FeatureVersion = SCHEMA.getField("featureVersion"); + + public Aggregation() { + super(new DataMap(8, 0.75F), SCHEMA, 6); + addChangeListener(__changeListener); + } + + public Aggregation(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static Aggregation.Fields fields() { + return _fields; + } + + public static Aggregation.ProjectionMask createMask() { + return new Aggregation.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for id + * + * @see Aggregation.Fields#id + */ + public boolean hasId() { + if (_idField!= null) { + return true; + } + return super._map.containsKey("id"); + } + + /** + * Remover for id + * + * @see Aggregation.Fields#id + */ + public void removeId() { + super._map.remove("id"); + } + + /** + * Getter for id + * + * @see Aggregation.Fields#id + */ + public Integer getId(GetMode mode) { + switch (mode) { + case STRICT: + return getId(); + case DEFAULT: + case NULL: + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for id + * + * @return + * Required field. Could be null for partial record. + * @see Aggregation.Fields#id + */ + @Nonnull + public Integer getId() { + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("id"); + } + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + + /** + * Setter for id + * + * @see Aggregation.Fields#id + */ + public Aggregation setId(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setId(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.Aggregation"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeId(); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + } + return this; + } + + /** + * Setter for id + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Aggregation.Fields#id + */ + public Aggregation setId( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.Aggregation to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + return this; + } + + /** + * Setter for id + * + * @see Aggregation.Fields#id + */ + public Aggregation setId(int value) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + return this; + } + + /** + * Existence checker for concreteKey + * + * @see Aggregation.Fields#concreteKey + */ + public boolean hasConcreteKey() { + if (_concreteKeyField!= null) { + return true; + } + return super._map.containsKey("concreteKey"); + } + + /** + * Remover for concreteKey + * + * @see Aggregation.Fields#concreteKey + */ + public void removeConcreteKey() { + super._map.remove("concreteKey"); + } + + /** + * Getter for concreteKey + * + * @see Aggregation.Fields#concreteKey + */ + public ConcreteKey getConcreteKey(GetMode mode) { + return getConcreteKey(); + } + + /** + * Getter for concreteKey + * + * @return + * Optional field. Always check for null. + * @see Aggregation.Fields#concreteKey + */ + @Nullable + public ConcreteKey getConcreteKey() { + if (_concreteKeyField!= null) { + return _concreteKeyField; + } else { + Object __rawValue = super._map.get("concreteKey"); + _concreteKeyField = ((__rawValue == null)?null:new ConcreteKey(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _concreteKeyField; + } + } + + /** + * Setter for concreteKey + * + * @see Aggregation.Fields#concreteKey + */ + public Aggregation setConcreteKey(ConcreteKey value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setConcreteKey(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeConcreteKey(); + } else { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + break; + } + return this; + } + + /** + * Setter for concreteKey + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Aggregation.Fields#concreteKey + */ + public Aggregation setConcreteKey( + @Nonnull + ConcreteKey value) { + if (value == null) { + throw new NullPointerException("Cannot set field concreteKey of com.linkedin.feathr.compute.Aggregation to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + return this; + } + + /** + * Existence checker for input + * + * @see Aggregation.Fields#input + */ + public boolean hasInput() { + if (_inputField!= null) { + return true; + } + return super._map.containsKey("input"); + } + + /** + * Remover for input + * + * @see Aggregation.Fields#input + */ + public void removeInput() { + super._map.remove("input"); + } + + /** + * Getter for input + * + * @see Aggregation.Fields#input + */ + public NodeReference getInput(GetMode mode) { + switch (mode) { + case STRICT: + return getInput(); + case DEFAULT: + case NULL: + if (_inputField!= null) { + return _inputField; + } else { + Object __rawValue = super._map.get("input"); + _inputField = ((__rawValue == null)?null:new NodeReference(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _inputField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for input + * + * @return + * Required field. Could be null for partial record. + * @see Aggregation.Fields#input + */ + @Nonnull + public NodeReference getInput() { + if (_inputField!= null) { + return _inputField; + } else { + Object __rawValue = super._map.get("input"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("input"); + } + _inputField = ((__rawValue == null)?null:new NodeReference(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _inputField; + } + } + + /** + * Setter for input + * + * @see Aggregation.Fields#input + */ + public Aggregation setInput(NodeReference value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setInput(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field input of com.linkedin.feathr.compute.Aggregation"); + } else { + CheckedUtil.putWithoutChecking(super._map, "input", value.data()); + _inputField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeInput(); + } else { + CheckedUtil.putWithoutChecking(super._map, "input", value.data()); + _inputField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "input", value.data()); + _inputField = value; + } + break; + } + return this; + } + + /** + * Setter for input + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Aggregation.Fields#input + */ + public Aggregation setInput( + @Nonnull + NodeReference value) { + if (value == null) { + throw new NullPointerException("Cannot set field input of com.linkedin.feathr.compute.Aggregation to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "input", value.data()); + _inputField = value; + } + return this; + } + + /** + * Existence checker for function + * + * @see Aggregation.Fields#function + */ + public boolean hasFunction() { + if (_functionField!= null) { + return true; + } + return super._map.containsKey("function"); + } + + /** + * Remover for function + * + * @see Aggregation.Fields#function + */ + public void removeFunction() { + super._map.remove("function"); + } + + /** + * Getter for function + * + * @see Aggregation.Fields#function + */ + public AggregationFunction getFunction(GetMode mode) { + switch (mode) { + case STRICT: + return getFunction(); + case DEFAULT: + case NULL: + if (_functionField!= null) { + return _functionField; + } else { + Object __rawValue = super._map.get("function"); + _functionField = ((__rawValue == null)?null:new AggregationFunction(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _functionField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for function + * + * @return + * Required field. Could be null for partial record. + * @see Aggregation.Fields#function + */ + @Nonnull + public AggregationFunction getFunction() { + if (_functionField!= null) { + return _functionField; + } else { + Object __rawValue = super._map.get("function"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("function"); + } + _functionField = ((__rawValue == null)?null:new AggregationFunction(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _functionField; + } + } + + /** + * Setter for function + * + * @see Aggregation.Fields#function + */ + public Aggregation setFunction(AggregationFunction value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFunction(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field function of com.linkedin.feathr.compute.Aggregation"); + } else { + CheckedUtil.putWithoutChecking(super._map, "function", value.data()); + _functionField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeFunction(); + } else { + CheckedUtil.putWithoutChecking(super._map, "function", value.data()); + _functionField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "function", value.data()); + _functionField = value; + } + break; + } + return this; + } + + /** + * Setter for function + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Aggregation.Fields#function + */ + public Aggregation setFunction( + @Nonnull + AggregationFunction value) { + if (value == null) { + throw new NullPointerException("Cannot set field function of com.linkedin.feathr.compute.Aggregation to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "function", value.data()); + _functionField = value; + } + return this; + } + + /** + * Existence checker for featureName + * + * @see Aggregation.Fields#featureName + */ + public boolean hasFeatureName() { + if (_featureNameField!= null) { + return true; + } + return super._map.containsKey("featureName"); + } + + /** + * Remover for featureName + * + * @see Aggregation.Fields#featureName + */ + public void removeFeatureName() { + super._map.remove("featureName"); + } + + /** + * Getter for featureName + * + * @see Aggregation.Fields#featureName + */ + public String getFeatureName(GetMode mode) { + switch (mode) { + case STRICT: + return getFeatureName(); + case DEFAULT: + case NULL: + if (_featureNameField!= null) { + return _featureNameField; + } else { + Object __rawValue = super._map.get("featureName"); + _featureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _featureNameField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for featureName + * + * @return + * Required field. Could be null for partial record. + * @see Aggregation.Fields#featureName + */ + @Nonnull + public String getFeatureName() { + if (_featureNameField!= null) { + return _featureNameField; + } else { + Object __rawValue = super._map.get("featureName"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("featureName"); + } + _featureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _featureNameField; + } + } + + /** + * Setter for featureName + * + * @see Aggregation.Fields#featureName + */ + public Aggregation setFeatureName(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFeatureName(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field featureName of com.linkedin.feathr.compute.Aggregation"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureName", value); + _featureNameField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeFeatureName(); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureName", value); + _featureNameField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "featureName", value); + _featureNameField = value; + } + break; + } + return this; + } + + /** + * Setter for featureName + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Aggregation.Fields#featureName + */ + public Aggregation setFeatureName( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field featureName of com.linkedin.feathr.compute.Aggregation to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureName", value); + _featureNameField = value; + } + return this; + } + + /** + * Existence checker for featureVersion + * + * @see Aggregation.Fields#featureVersion + */ + public boolean hasFeatureVersion() { + if (_featureVersionField!= null) { + return true; + } + return super._map.containsKey("featureVersion"); + } + + /** + * Remover for featureVersion + * + * @see Aggregation.Fields#featureVersion + */ + public void removeFeatureVersion() { + super._map.remove("featureVersion"); + } + + /** + * Getter for featureVersion + * + * @see Aggregation.Fields#featureVersion + */ + public FeatureVersion getFeatureVersion(GetMode mode) { + switch (mode) { + case STRICT: + return getFeatureVersion(); + case DEFAULT: + case NULL: + if (_featureVersionField!= null) { + return _featureVersionField; + } else { + Object __rawValue = super._map.get("featureVersion"); + _featureVersionField = ((__rawValue == null)?null:new FeatureVersion(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _featureVersionField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for featureVersion + * + * @return + * Required field. Could be null for partial record. + * @see Aggregation.Fields#featureVersion + */ + @Nonnull + public FeatureVersion getFeatureVersion() { + if (_featureVersionField!= null) { + return _featureVersionField; + } else { + Object __rawValue = super._map.get("featureVersion"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("featureVersion"); + } + _featureVersionField = ((__rawValue == null)?null:new FeatureVersion(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _featureVersionField; + } + } + + /** + * Setter for featureVersion + * + * @see Aggregation.Fields#featureVersion + */ + public Aggregation setFeatureVersion(FeatureVersion value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFeatureVersion(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field featureVersion of com.linkedin.feathr.compute.Aggregation"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); + _featureVersionField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeFeatureVersion(); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); + _featureVersionField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); + _featureVersionField = value; + } + break; + } + return this; + } + + /** + * Setter for featureVersion + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Aggregation.Fields#featureVersion + */ + public Aggregation setFeatureVersion( + @Nonnull + FeatureVersion value) { + if (value == null) { + throw new NullPointerException("Cannot set field featureVersion of com.linkedin.feathr.compute.Aggregation to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); + _featureVersionField = value; + } + return this; + } + + @Override + public Aggregation clone() + throws CloneNotSupportedException + { + Aggregation __clone = ((Aggregation) super.clone()); + __clone.__changeListener = new Aggregation.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public Aggregation copy() + throws CloneNotSupportedException + { + Aggregation __copy = ((Aggregation) super.copy()); + __copy._inputField = null; + __copy._featureNameField = null; + __copy._functionField = null; + __copy._idField = null; + __copy._concreteKeyField = null; + __copy._featureVersionField = null; + __copy.__changeListener = new Aggregation.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final Aggregation __objectRef; + + private ChangeListener(Aggregation reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "input": + __objectRef._inputField = null; + break; + case "featureName": + __objectRef._featureNameField = null; + break; + case "function": + __objectRef._functionField = null; + break; + case "id": + __objectRef._idField = null; + break; + case "concreteKey": + __objectRef._concreteKeyField = null; + break; + case "featureVersion": + __objectRef._featureVersionField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The node would be represented by this id. + * + */ + public PathSpec id() { + return new PathSpec(getPathComponents(), "id"); + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public com.linkedin.feathr.compute.ConcreteKey.Fields concreteKey() { + return new com.linkedin.feathr.compute.ConcreteKey.Fields(getPathComponents(), "concreteKey"); + } + + /** + * The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node. + * + */ + public com.linkedin.feathr.compute.NodeReference.Fields input() { + return new com.linkedin.feathr.compute.NodeReference.Fields(getPathComponents(), "input"); + } + + /** + * All the aggregation related parameters and functions are bundled into this. + * + */ + public com.linkedin.feathr.compute.AggregationFunction.Fields function() { + return new com.linkedin.feathr.compute.AggregationFunction.Fields(getPathComponents(), "function"); + } + + /** + * If the node is representing a feature, the feature name should be associated with the node. + * + */ + public PathSpec featureName() { + return new PathSpec(getPathComponents(), "featureName"); + } + + /** + * feature version of the feature + * + */ + public com.linkedin.feathr.compute.FeatureVersion.Fields featureVersion() { + return new com.linkedin.feathr.compute.FeatureVersion.Fields(getPathComponents(), "featureVersion"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.ConcreteKey.ProjectionMask _concreteKeyMask; + private com.linkedin.feathr.compute.NodeReference.ProjectionMask _inputMask; + private com.linkedin.feathr.compute.AggregationFunction.ProjectionMask _functionMask; + private com.linkedin.feathr.compute.FeatureVersion.ProjectionMask _featureVersionMask; + + ProjectionMask() { + super(8); + } + + /** + * The node would be represented by this id. + * + */ + public Aggregation.ProjectionMask withId() { + getDataMap().put("id", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public Aggregation.ProjectionMask withConcreteKey(Function nestedMask) { + _concreteKeyMask = nestedMask.apply(((_concreteKeyMask == null)?ConcreteKey.createMask():_concreteKeyMask)); + getDataMap().put("concreteKey", _concreteKeyMask.getDataMap()); + return this; + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public Aggregation.ProjectionMask withConcreteKey() { + _concreteKeyMask = null; + getDataMap().put("concreteKey", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node. + * + */ + public Aggregation.ProjectionMask withInput(Function nestedMask) { + _inputMask = nestedMask.apply(((_inputMask == null)?NodeReference.createMask():_inputMask)); + getDataMap().put("input", _inputMask.getDataMap()); + return this; + } + + /** + * The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node. + * + */ + public Aggregation.ProjectionMask withInput() { + _inputMask = null; + getDataMap().put("input", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * All the aggregation related parameters and functions are bundled into this. + * + */ + public Aggregation.ProjectionMask withFunction(Function nestedMask) { + _functionMask = nestedMask.apply(((_functionMask == null)?AggregationFunction.createMask():_functionMask)); + getDataMap().put("function", _functionMask.getDataMap()); + return this; + } + + /** + * All the aggregation related parameters and functions are bundled into this. + * + */ + public Aggregation.ProjectionMask withFunction() { + _functionMask = null; + getDataMap().put("function", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * If the node is representing a feature, the feature name should be associated with the node. + * + */ + public Aggregation.ProjectionMask withFeatureName() { + getDataMap().put("featureName", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * feature version of the feature + * + */ + public Aggregation.ProjectionMask withFeatureVersion(Function nestedMask) { + _featureVersionMask = nestedMask.apply(((_featureVersionMask == null)?FeatureVersion.createMask():_featureVersionMask)); + getDataMap().put("featureVersion", _featureVersionMask.getDataMap()); + return this; + } + + /** + * feature version of the feature + * + */ + public Aggregation.ProjectionMask withFeatureVersion() { + _featureVersionMask = null; + getDataMap().put("featureVersion", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationFunction.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationFunction.java new file mode 100644 index 000000000..ab0933539 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationFunction.java @@ -0,0 +1,394 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.StringMap; + + +/** + * All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\AggregationFunction.pdl.") +public class AggregationFunction + extends RecordTemplate +{ + + private final static AggregationFunction.Fields _fields = new AggregationFunction.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node.*/record AggregationFunction{/**The aggregation function.*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**All the aggregation parameters should be bundled into this map. For now, the possible parameters are:-\r\na. target_column - Aggregation column\r\nb. window_size - aggregation window size\r\nc. window unit - aggregation window unit (ex - day, hour)\r\nd. lateral_view_expression - definition of a lateral view for the feature.\r\ne. lateral_view_table_alias - An alias for the lateral view\r\nf. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression.\r\ng. groupBy - groupBy columns. Should be a sparkSql expression.*/parameters:optional map[string,string]}", SchemaFormatType.PDL)); + private String _operatorField = null; + private StringMap _parametersField = null; + private AggregationFunction.ChangeListener __changeListener = new AggregationFunction.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Operator = SCHEMA.getField("operator"); + private final static RecordDataSchema.Field FIELD_Parameters = SCHEMA.getField("parameters"); + + public AggregationFunction() { + super(new DataMap(3, 0.75F), SCHEMA, 2); + addChangeListener(__changeListener); + } + + public AggregationFunction(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static AggregationFunction.Fields fields() { + return _fields; + } + + public static AggregationFunction.ProjectionMask createMask() { + return new AggregationFunction.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for operator + * + * @see AggregationFunction.Fields#operator + */ + public boolean hasOperator() { + if (_operatorField!= null) { + return true; + } + return super._map.containsKey("operator"); + } + + /** + * Remover for operator + * + * @see AggregationFunction.Fields#operator + */ + public void removeOperator() { + super._map.remove("operator"); + } + + /** + * Getter for operator + * + * @see AggregationFunction.Fields#operator + */ + public String getOperator(GetMode mode) { + switch (mode) { + case STRICT: + return getOperator(); + case DEFAULT: + case NULL: + if (_operatorField!= null) { + return _operatorField; + } else { + Object __rawValue = super._map.get("operator"); + _operatorField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _operatorField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for operator + * + * @return + * Required field. Could be null for partial record. + * @see AggregationFunction.Fields#operator + */ + @Nonnull + public String getOperator() { + if (_operatorField!= null) { + return _operatorField; + } else { + Object __rawValue = super._map.get("operator"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("operator"); + } + _operatorField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _operatorField; + } + } + + /** + * Setter for operator + * + * @see AggregationFunction.Fields#operator + */ + public AggregationFunction setOperator(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setOperator(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field operator of com.linkedin.feathr.compute.AggregationFunction"); + } else { + CheckedUtil.putWithoutChecking(super._map, "operator", value); + _operatorField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeOperator(); + } else { + CheckedUtil.putWithoutChecking(super._map, "operator", value); + _operatorField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "operator", value); + _operatorField = value; + } + break; + } + return this; + } + + /** + * Setter for operator + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see AggregationFunction.Fields#operator + */ + public AggregationFunction setOperator( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field operator of com.linkedin.feathr.compute.AggregationFunction to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "operator", value); + _operatorField = value; + } + return this; + } + + /** + * Existence checker for parameters + * + * @see AggregationFunction.Fields#parameters + */ + public boolean hasParameters() { + if (_parametersField!= null) { + return true; + } + return super._map.containsKey("parameters"); + } + + /** + * Remover for parameters + * + * @see AggregationFunction.Fields#parameters + */ + public void removeParameters() { + super._map.remove("parameters"); + } + + /** + * Getter for parameters + * + * @see AggregationFunction.Fields#parameters + */ + public StringMap getParameters(GetMode mode) { + return getParameters(); + } + + /** + * Getter for parameters + * + * @return + * Optional field. Always check for null. + * @see AggregationFunction.Fields#parameters + */ + @Nullable + public StringMap getParameters() { + if (_parametersField!= null) { + return _parametersField; + } else { + Object __rawValue = super._map.get("parameters"); + _parametersField = ((__rawValue == null)?null:new StringMap(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _parametersField; + } + } + + /** + * Setter for parameters + * + * @see AggregationFunction.Fields#parameters + */ + public AggregationFunction setParameters(StringMap value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setParameters(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeParameters(); + } else { + CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); + _parametersField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); + _parametersField = value; + } + break; + } + return this; + } + + /** + * Setter for parameters + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see AggregationFunction.Fields#parameters + */ + public AggregationFunction setParameters( + @Nonnull + StringMap value) { + if (value == null) { + throw new NullPointerException("Cannot set field parameters of com.linkedin.feathr.compute.AggregationFunction to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); + _parametersField = value; + } + return this; + } + + @Override + public AggregationFunction clone() + throws CloneNotSupportedException + { + AggregationFunction __clone = ((AggregationFunction) super.clone()); + __clone.__changeListener = new AggregationFunction.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public AggregationFunction copy() + throws CloneNotSupportedException + { + AggregationFunction __copy = ((AggregationFunction) super.copy()); + __copy._parametersField = null; + __copy._operatorField = null; + __copy.__changeListener = new AggregationFunction.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final AggregationFunction __objectRef; + + private ChangeListener(AggregationFunction reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "parameters": + __objectRef._parametersField = null; + break; + case "operator": + __objectRef._operatorField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The aggregation function. + * + */ + public PathSpec operator() { + return new PathSpec(getPathComponents(), "operator"); + } + + /** + * All the aggregation parameters should be bundled into this map. For now, the possible parameters are:- + * a. target_column - Aggregation column + * b. window_size - aggregation window size + * c. window unit - aggregation window unit (ex - day, hour) + * d. lateral_view_expression - definition of a lateral view for the feature. + * e. lateral_view_table_alias - An alias for the lateral view + * f. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression. + * g. groupBy - groupBy columns. Should be a sparkSql expression. + * + */ + public PathSpec parameters() { + return new PathSpec(getPathComponents(), "parameters"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(3); + } + + /** + * The aggregation function. + * + */ + public AggregationFunction.ProjectionMask withOperator() { + getDataMap().put("operator", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * All the aggregation parameters should be bundled into this map. For now, the possible parameters are:- + * a. target_column - Aggregation column + * b. window_size - aggregation window size + * c. window unit - aggregation window unit (ex - day, hour) + * d. lateral_view_expression - definition of a lateral view for the feature. + * e. lateral_view_table_alias - An alias for the lateral view + * f. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression. + * g. groupBy - groupBy columns. Should be a sparkSql expression. + * + */ + public AggregationFunction.ProjectionMask withParameters() { + getDataMap().put("parameters", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationType.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationType.java new file mode 100644 index 000000000..fc3cc0284 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationType.java @@ -0,0 +1,78 @@ + +package com.linkedin.feathr.compute; + +import javax.annotation.Generated; +import com.linkedin.data.schema.EnumDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; + + +/** + * + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SlidingWindowFeature.pdl.") +public enum AggregationType { + + + /** + * Sum. + * + */ + SUM, + + /** + * Count. + * + */ + COUNT, + + /** + * Max. + * + */ + MAX, + + /** + * Min. + * + */ + MIN, + + /** + * Average. + * + */ + AVG, + + /** + * Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Max pooling is done by applying a max filter to (usually) non-overlapping subregions of the initial representation. + * + */ + MAX_POOLING, + + /** + * Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Min pooling is done by applying a min filter to (usually) non-overlapping subregions of the initial representation. + * + */ + MIN_POOLING, + + /** + * Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Average pooling is done by applying a average filter to (usually) non-overlapping subregions of the initial representation. + * + */ + AVG_POOLING, + + /** + * Latest + * + */ + LATEST, + $UNKNOWN; + private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute,enum AggregationType{/** Sum. */SUM/** Count. */COUNT/** Max. */MAX/** Min. */MIN/** Average. */AVG/** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Max pooling is done by applying a max filter to (usually) non-overlapping subregions of the initial representation. */MAX_POOLING/** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Min pooling is done by applying a min filter to (usually) non-overlapping subregions of the initial representation. */MIN_POOLING/** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Average pooling is done by applying a average filter to (usually) non-overlapping subregions of the initial representation. */AVG_POOLING/** Latest */LATEST}", SchemaFormatType.PDL)); + + public static EnumDataSchema dataSchema() { + return SCHEMA; + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNode.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNode.java new file mode 100644 index 000000000..fe1ec5a8f --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNode.java @@ -0,0 +1,357 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.DataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.TyperefDataSchema; +import com.linkedin.data.schema.UnionDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.HasTyperefInfo; +import com.linkedin.data.template.TyperefInfo; +import com.linkedin.data.template.UnionTemplate; + +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\AnyNode.pdl.") +public class AnyNode + extends UnionTemplate + implements HasTyperefInfo +{ + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**A node to represent an aggregation step. The aggregation inputs like the groupBy field, agg function are delegated to [[AggregationFunction]].\r\nThis node can represent a feature. As of now, in this step we will be using the SWA library from Spark-algorithms.*/record Aggregation includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node.*/input:/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}/**All the aggregation related parameters and functions are bundled into this.*/function:/**All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node.*/record AggregationFunction{/**The aggregation function.*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**All the aggregation parameters should be bundled into this map. For now, the possible parameters are:-\r\na. target_column - Aggregation column\r\nb. window_size - aggregation window size\r\nc. window unit - aggregation window unit (ex - day, hour)\r\nd. lateral_view_expression - definition of a lateral view for the feature.\r\ne. lateral_view_table_alias - An alias for the lateral view\r\nf. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression.\r\ng. groupBy - groupBy columns. Should be a sparkSql expression.*/parameters:optional map[string,string]}/**If the node is representing a feature, the feature name should be associated with the node.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}}{namespace com.linkedin.feathr.compute/**Representation of the datasource node. There are 3 types of datasource nodes:-\r\nContext - To represent the observation data entities (like the join key or passthrough feature columns)\r\nUpdate - To represent a non-timepartitioned datasource node.\r\nEvent - To represent a time-partitioned datasource node.\r\n\r\nTODO - Maybe, it makes sense more sense to refactor it by make this an abstract object, and deriving the three different nodes from it.*/record DataSource includes AbstractNode{/**Type of node, ie - Context, Update, Event*/sourceType:/**Type of datasource node.*/enum DataSourceType{/**Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log.*/UPDATE/**Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K\u00e2\u20ac\u2122d)\r\nover a limited window of time.*/EVENT/**Reprent the observation data entities (like the join key or passthrough feature columns)*/CONTEXT}/**for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI.*/externalSourceRef:string/**Raw key expression as entered by the user. This hocon parsing happens at the execution engine side.*/keyExpression:string/**mvel or spark or user-defined class*/keyExpressionType:/**Different key formats supported.\r\nTodo - We probably do not want to generalize this as a kind of key-operator in the core compute model,\r\nwith instances such as for MVEL or SQL being available (e.g. via an OperatorId reference).*/enum KeyExpressionType{/**Java-based MVEL*/MVEL/**Spark-SQL*/SQL/**Custom java/scala UDF*/UDF}/**File partition format.*/filePartitionFormat:optional string/**Timestamp column info, to be available only for an event datasource node.*/timestampColumnInfo:optional/**Representation of a timestamp column field*/record TimestampCol{/**Timestamp column expression.*/expression:string/**Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis*/format:string}}}{namespace com.linkedin.feathr.compute/**A node to represent a feature which is to be computed by using an already computed feature as the key.\r\nhttps://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?spaceKey=ENGS&title=feathr+Offline+User+Guide#FrameOfflineUserGuide-sequentialjoin*/record Lookup includes AbstractNode{/**An array of references to a node and keys.\r\n\r\nFor now, we do not support lookup of just a key reference, but we have added that as a placeholder.\r\n\r\nA node reference consists of node id and a key reference.\r\nIn sequential join the lookup key would be a combination of the\r\nfeature node representing the base feature (lookup node) and the key associated with it. For example,:-\r\nseqJoinFeature: {\r\n base: {key: x, feature: baseFeature}\r\n expansion: {key: y, feature: expansionFeature}\r\n aggregation: UNION\r\n}\r\nHere, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would\r\npoint to the index of \"x\" in the key array of baseFeature.*/lookupKey:array[union[NodeReference,KeyReference]]/**The node id of the node containing the expansion feature.*/lookupNode:NodeId/**Aggregation type as listed in\r\nhttps://jarvis.corp.linkedin.com/codesearch/result/\r\n?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7\r\n*/aggregation:string/**feature name of the feature which would be computed.\r\nwe need feature name here for 2 main reasons.\r\n1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and\r\nwe want to leverage that.\r\n2. For default values. Similar to above, there are existing APIs which create default value map from feature name ->\r\ndefault value.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}}{namespace com.linkedin.feathr.compute/**Representation of a transformation node.*/record Transformation includes AbstractNode{/**An array of node references which should be considered as input to apply the transformation function.*/inputs:array[NodeReference]/**The transformation function.*/function:/**The transformation function*/record TransformationFunction{/**Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class.\r\n*/operator:OperatorId/**The various attributes required to represent the transformation function are captured in a map format.\r\nFor example, mvel expression or java udf class name*/parameters:optional map[string,string]}/**Feature name here is used so we retain feature name, type, and default values even after graph is resolved.\r\nFeature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}}{namespace com.linkedin.feathr.compute/**A temporary node which would exist only while parsing the graph. For example, when parsing an object if there is a reference to a feature\r\nname, we will create an external node. This would get resolved later in the computation.*/record External includes AbstractNode{/**Name of the external object it should refer to.*/name:string}}]", SchemaFormatType.PDL)); + private com.linkedin.feathr.compute.Aggregation _aggregationMember = null; + private com.linkedin.feathr.compute.DataSource _dataSourceMember = null; + private com.linkedin.feathr.compute.Lookup _lookupMember = null; + private com.linkedin.feathr.compute.Transformation _transformationMember = null; + private com.linkedin.feathr.compute.External _externalMember = null; + private AnyNode.ChangeListener __changeListener = new AnyNode.ChangeListener(this); + private final static DataSchema MEMBER_Aggregation = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.Aggregation"); + private final static DataSchema MEMBER_DataSource = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.DataSource"); + private final static DataSchema MEMBER_Lookup = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.Lookup"); + private final static DataSchema MEMBER_Transformation = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.Transformation"); + private final static DataSchema MEMBER_External = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.External"); + private final static TyperefInfo TYPEREFINFO = new AnyNode.UnionTyperefInfo(); + + public AnyNode() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public AnyNode(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static AnyNode create(com.linkedin.feathr.compute.Aggregation value) { + AnyNode newUnion = new AnyNode(); + newUnion.setAggregation(value); + return newUnion; + } + + public boolean isAggregation() { + return memberIs("com.linkedin.feathr.compute.Aggregation"); + } + + public com.linkedin.feathr.compute.Aggregation getAggregation() { + checkNotNull(); + if (_aggregationMember!= null) { + return _aggregationMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.Aggregation"); + _aggregationMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.Aggregation(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _aggregationMember; + } + + public void setAggregation(com.linkedin.feathr.compute.Aggregation value) { + checkNotNull(); + super._map.clear(); + _aggregationMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.Aggregation", value.data()); + } + + public static AnyNode create(com.linkedin.feathr.compute.DataSource value) { + AnyNode newUnion = new AnyNode(); + newUnion.setDataSource(value); + return newUnion; + } + + public boolean isDataSource() { + return memberIs("com.linkedin.feathr.compute.DataSource"); + } + + public com.linkedin.feathr.compute.DataSource getDataSource() { + checkNotNull(); + if (_dataSourceMember!= null) { + return _dataSourceMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.DataSource"); + _dataSourceMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.DataSource(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _dataSourceMember; + } + + public void setDataSource(com.linkedin.feathr.compute.DataSource value) { + checkNotNull(); + super._map.clear(); + _dataSourceMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.DataSource", value.data()); + } + + public static AnyNode create(com.linkedin.feathr.compute.Lookup value) { + AnyNode newUnion = new AnyNode(); + newUnion.setLookup(value); + return newUnion; + } + + public boolean isLookup() { + return memberIs("com.linkedin.feathr.compute.Lookup"); + } + + public com.linkedin.feathr.compute.Lookup getLookup() { + checkNotNull(); + if (_lookupMember!= null) { + return _lookupMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.Lookup"); + _lookupMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.Lookup(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _lookupMember; + } + + public void setLookup(com.linkedin.feathr.compute.Lookup value) { + checkNotNull(); + super._map.clear(); + _lookupMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.Lookup", value.data()); + } + + public static AnyNode create(com.linkedin.feathr.compute.Transformation value) { + AnyNode newUnion = new AnyNode(); + newUnion.setTransformation(value); + return newUnion; + } + + public boolean isTransformation() { + return memberIs("com.linkedin.feathr.compute.Transformation"); + } + + public com.linkedin.feathr.compute.Transformation getTransformation() { + checkNotNull(); + if (_transformationMember!= null) { + return _transformationMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.Transformation"); + _transformationMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.Transformation(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _transformationMember; + } + + public void setTransformation(com.linkedin.feathr.compute.Transformation value) { + checkNotNull(); + super._map.clear(); + _transformationMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.Transformation", value.data()); + } + + public static AnyNode create(com.linkedin.feathr.compute.External value) { + AnyNode newUnion = new AnyNode(); + newUnion.setExternal(value); + return newUnion; + } + + public boolean isExternal() { + return memberIs("com.linkedin.feathr.compute.External"); + } + + public com.linkedin.feathr.compute.External getExternal() { + checkNotNull(); + if (_externalMember!= null) { + return _externalMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.External"); + _externalMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.External(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _externalMember; + } + + public void setExternal(com.linkedin.feathr.compute.External value) { + checkNotNull(); + super._map.clear(); + _externalMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.External", value.data()); + } + + public static AnyNode.ProjectionMask createMask() { + return new AnyNode.ProjectionMask(); + } + + @Override + public AnyNode clone() + throws CloneNotSupportedException + { + AnyNode __clone = ((AnyNode) super.clone()); + __clone.__changeListener = new AnyNode.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public AnyNode copy() + throws CloneNotSupportedException + { + AnyNode __copy = ((AnyNode) super.copy()); + __copy._lookupMember = null; + __copy._transformationMember = null; + __copy._aggregationMember = null; + __copy._dataSourceMember = null; + __copy._externalMember = null; + __copy.__changeListener = new AnyNode.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + public TyperefInfo typerefInfo() { + return TYPEREFINFO; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final AnyNode __objectRef; + + private ChangeListener(AnyNode reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "com.linkedin.feathr.compute.Lookup": + __objectRef._lookupMember = null; + break; + case "com.linkedin.feathr.compute.Transformation": + __objectRef._transformationMember = null; + break; + case "com.linkedin.feathr.compute.Aggregation": + __objectRef._aggregationMember = null; + break; + case "com.linkedin.feathr.compute.DataSource": + __objectRef._dataSourceMember = null; + break; + case "com.linkedin.feathr.compute.External": + __objectRef._externalMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.compute.Aggregation.Fields Aggregation() { + return new com.linkedin.feathr.compute.Aggregation.Fields(getPathComponents(), "com.linkedin.feathr.compute.Aggregation"); + } + + public com.linkedin.feathr.compute.DataSource.Fields DataSource() { + return new com.linkedin.feathr.compute.DataSource.Fields(getPathComponents(), "com.linkedin.feathr.compute.DataSource"); + } + + public com.linkedin.feathr.compute.Lookup.Fields Lookup() { + return new com.linkedin.feathr.compute.Lookup.Fields(getPathComponents(), "com.linkedin.feathr.compute.Lookup"); + } + + public com.linkedin.feathr.compute.Transformation.Fields Transformation() { + return new com.linkedin.feathr.compute.Transformation.Fields(getPathComponents(), "com.linkedin.feathr.compute.Transformation"); + } + + public com.linkedin.feathr.compute.External.Fields External() { + return new com.linkedin.feathr.compute.External.Fields(getPathComponents(), "com.linkedin.feathr.compute.External"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.Aggregation.ProjectionMask _AggregationMask; + private com.linkedin.feathr.compute.DataSource.ProjectionMask _DataSourceMask; + private com.linkedin.feathr.compute.Lookup.ProjectionMask _LookupMask; + private com.linkedin.feathr.compute.Transformation.ProjectionMask _TransformationMask; + private com.linkedin.feathr.compute.External.ProjectionMask _ExternalMask; + + ProjectionMask() { + super(7); + } + + public AnyNode.ProjectionMask withAggregation(Function nestedMask) { + _AggregationMask = nestedMask.apply(((_AggregationMask == null)?com.linkedin.feathr.compute.Aggregation.createMask():_AggregationMask)); + getDataMap().put("com.linkedin.feathr.compute.Aggregation", _AggregationMask.getDataMap()); + return this; + } + + public AnyNode.ProjectionMask withDataSource(Function nestedMask) { + _DataSourceMask = nestedMask.apply(((_DataSourceMask == null)?com.linkedin.feathr.compute.DataSource.createMask():_DataSourceMask)); + getDataMap().put("com.linkedin.feathr.compute.DataSource", _DataSourceMask.getDataMap()); + return this; + } + + public AnyNode.ProjectionMask withLookup(Function nestedMask) { + _LookupMask = nestedMask.apply(((_LookupMask == null)?com.linkedin.feathr.compute.Lookup.createMask():_LookupMask)); + getDataMap().put("com.linkedin.feathr.compute.Lookup", _LookupMask.getDataMap()); + return this; + } + + public AnyNode.ProjectionMask withTransformation(Function nestedMask) { + _TransformationMask = nestedMask.apply(((_TransformationMask == null)?com.linkedin.feathr.compute.Transformation.createMask():_TransformationMask)); + getDataMap().put("com.linkedin.feathr.compute.Transformation", _TransformationMask.getDataMap()); + return this; + } + + public AnyNode.ProjectionMask withExternal(Function nestedMask) { + _ExternalMask = nestedMask.apply(((_ExternalMask == null)?com.linkedin.feathr.compute.External.createMask():_ExternalMask)); + getDataMap().put("com.linkedin.feathr.compute.External", _ExternalMask.getDataMap()); + return this; + } + + } + + + /** + * A typeref for all the different types of nodes. + * + */ + private final static class UnionTyperefInfo + extends TyperefInfo + { + + private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**A typeref for all the different types of nodes.*/typeref AnyNode=union[/**A node to represent an aggregation step. The aggregation inputs like the groupBy field, agg function are delegated to [[AggregationFunction]].\r\nThis node can represent a feature. As of now, in this step we will be using the SWA library from Spark-algorithms.*/record Aggregation includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node.*/input:/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}/**All the aggregation related parameters and functions are bundled into this.*/function:/**All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node.*/record AggregationFunction{/**The aggregation function.*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**All the aggregation parameters should be bundled into this map. For now, the possible parameters are:-\r\na. target_column - Aggregation column\r\nb. window_size - aggregation window size\r\nc. window unit - aggregation window unit (ex - day, hour)\r\nd. lateral_view_expression - definition of a lateral view for the feature.\r\ne. lateral_view_table_alias - An alias for the lateral view\r\nf. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression.\r\ng. groupBy - groupBy columns. Should be a sparkSql expression.*/parameters:optional map[string,string]}/**If the node is representing a feature, the feature name should be associated with the node.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}/**Representation of the datasource node. There are 3 types of datasource nodes:-\r\nContext - To represent the observation data entities (like the join key or passthrough feature columns)\r\nUpdate - To represent a non-timepartitioned datasource node.\r\nEvent - To represent a time-partitioned datasource node.\r\n\r\nTODO - Maybe, it makes sense more sense to refactor it by make this an abstract object, and deriving the three different nodes from it.*/record DataSource includes AbstractNode{/**Type of node, ie - Context, Update, Event*/sourceType:/**Type of datasource node.*/enum DataSourceType{/**Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log.*/UPDATE/**Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K\u00e2\u20ac\u2122d)\r\nover a limited window of time.*/EVENT/**Reprent the observation data entities (like the join key or passthrough feature columns)*/CONTEXT}/**for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI.*/externalSourceRef:string/**Raw key expression as entered by the user. This hocon parsing happens at the execution engine side.*/keyExpression:string/**mvel or spark or user-defined class*/keyExpressionType:/**Different key formats supported.\r\nTodo - We probably do not want to generalize this as a kind of key-operator in the core compute model,\r\nwith instances such as for MVEL or SQL being available (e.g. via an OperatorId reference).*/enum KeyExpressionType{/**Java-based MVEL*/MVEL/**Spark-SQL*/SQL/**Custom java/scala UDF*/UDF}/**File partition format.*/filePartitionFormat:optional string/**Timestamp column info, to be available only for an event datasource node.*/timestampColumnInfo:optional/**Representation of a timestamp column field*/record TimestampCol{/**Timestamp column expression.*/expression:string/**Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis*/format:string}}/**A node to represent a feature which is to be computed by using an already computed feature as the key.\r\nhttps://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?spaceKey=ENGS&title=feathr+Offline+User+Guide#FrameOfflineUserGuide-sequentialjoin*/record Lookup includes AbstractNode{/**An array of references to a node and keys.\r\n\r\nFor now, we do not support lookup of just a key reference, but we have added that as a placeholder.\r\n\r\nA node reference consists of node id and a key reference.\r\nIn sequential join the lookup key would be a combination of the\r\nfeature node representing the base feature (lookup node) and the key associated with it. For example,:-\r\nseqJoinFeature: {\r\n base: {key: x, feature: baseFeature}\r\n expansion: {key: y, feature: expansionFeature}\r\n aggregation: UNION\r\n}\r\nHere, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would\r\npoint to the index of \"x\" in the key array of baseFeature.*/lookupKey:array[union[NodeReference,KeyReference]]/**The node id of the node containing the expansion feature.*/lookupNode:NodeId/**Aggregation type as listed in\r\nhttps://jarvis.corp.linkedin.com/codesearch/result/\r\n?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7\r\n*/aggregation:string/**feature name of the feature which would be computed.\r\nwe need feature name here for 2 main reasons.\r\n1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and\r\nwe want to leverage that.\r\n2. For default values. Similar to above, there are existing APIs which create default value map from feature name ->\r\ndefault value.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}/**Representation of a transformation node.*/record Transformation includes AbstractNode{/**An array of node references which should be considered as input to apply the transformation function.*/inputs:array[NodeReference]/**The transformation function.*/function:/**The transformation function*/record TransformationFunction{/**Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class.\r\n*/operator:OperatorId/**The various attributes required to represent the transformation function are captured in a map format.\r\nFor example, mvel expression or java udf class name*/parameters:optional map[string,string]}/**Feature name here is used so we retain feature name, type, and default values even after graph is resolved.\r\nFeature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}/**A temporary node which would exist only while parsing the graph. For example, when parsing an object if there is a reference to a feature\r\nname, we will create an external node. This would get resolved later in the computation.*/record External includes AbstractNode{/**Name of the external object it should refer to.*/name:string}]", SchemaFormatType.PDL)); + + public UnionTyperefInfo() { + super(SCHEMA); + } + + public static TyperefDataSchema dataSchema() { + return SCHEMA; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNodeArray.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNodeArray.java new file mode 100644 index 000000000..92b608259 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNodeArray.java @@ -0,0 +1,117 @@ + +package com.linkedin.feathr.compute; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import com.linkedin.data.DataList; +import com.linkedin.data.schema.ArrayDataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.TemplateOutputCastException; +import com.linkedin.data.template.WrappingArrayTemplate; + +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\ComputeGraph.pdl.") +public class AnyNodeArray + extends WrappingArrayTemplate +{ + + private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[union[{namespace com.linkedin.feathr.compute/**A node to represent an aggregation step. The aggregation inputs like the groupBy field, agg function are delegated to [[AggregationFunction]].\r\nThis node can represent a feature. As of now, in this step we will be using the SWA library from Spark-algorithms.*/record Aggregation includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node.*/input:/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}/**All the aggregation related parameters and functions are bundled into this.*/function:/**All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node.*/record AggregationFunction{/**The aggregation function.*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**All the aggregation parameters should be bundled into this map. For now, the possible parameters are:-\r\na. target_column - Aggregation column\r\nb. window_size - aggregation window size\r\nc. window unit - aggregation window unit (ex - day, hour)\r\nd. lateral_view_expression - definition of a lateral view for the feature.\r\ne. lateral_view_table_alias - An alias for the lateral view\r\nf. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression.\r\ng. groupBy - groupBy columns. Should be a sparkSql expression.*/parameters:optional map[string,string]}/**If the node is representing a feature, the feature name should be associated with the node.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}}{namespace com.linkedin.feathr.compute/**Representation of the datasource node. There are 3 types of datasource nodes:-\r\nContext - To represent the observation data entities (like the join key or passthrough feature columns)\r\nUpdate - To represent a non-timepartitioned datasource node.\r\nEvent - To represent a time-partitioned datasource node.\r\n\r\nTODO - Maybe, it makes sense more sense to refactor it by make this an abstract object, and deriving the three different nodes from it.*/record DataSource includes AbstractNode{/**Type of node, ie - Context, Update, Event*/sourceType:/**Type of datasource node.*/enum DataSourceType{/**Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log.*/UPDATE/**Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K\u00e2\u20ac\u2122d)\r\nover a limited window of time.*/EVENT/**Reprent the observation data entities (like the join key or passthrough feature columns)*/CONTEXT}/**for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI.*/externalSourceRef:string/**Raw key expression as entered by the user. This hocon parsing happens at the execution engine side.*/keyExpression:string/**mvel or spark or user-defined class*/keyExpressionType:/**Different key formats supported.\r\nTodo - We probably do not want to generalize this as a kind of key-operator in the core compute model,\r\nwith instances such as for MVEL or SQL being available (e.g. via an OperatorId reference).*/enum KeyExpressionType{/**Java-based MVEL*/MVEL/**Spark-SQL*/SQL/**Custom java/scala UDF*/UDF}/**File partition format.*/filePartitionFormat:optional string/**Timestamp column info, to be available only for an event datasource node.*/timestampColumnInfo:optional/**Representation of a timestamp column field*/record TimestampCol{/**Timestamp column expression.*/expression:string/**Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis*/format:string}}}{namespace com.linkedin.feathr.compute/**A node to represent a feature which is to be computed by using an already computed feature as the key.\r\nhttps://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?spaceKey=ENGS&title=feathr+Offline+User+Guide#FrameOfflineUserGuide-sequentialjoin*/record Lookup includes AbstractNode{/**An array of references to a node and keys.\r\n\r\nFor now, we do not support lookup of just a key reference, but we have added that as a placeholder.\r\n\r\nA node reference consists of node id and a key reference.\r\nIn sequential join the lookup key would be a combination of the\r\nfeature node representing the base feature (lookup node) and the key associated with it. For example,:-\r\nseqJoinFeature: {\r\n base: {key: x, feature: baseFeature}\r\n expansion: {key: y, feature: expansionFeature}\r\n aggregation: UNION\r\n}\r\nHere, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would\r\npoint to the index of \"x\" in the key array of baseFeature.*/lookupKey:array[union[NodeReference,KeyReference]]/**The node id of the node containing the expansion feature.*/lookupNode:NodeId/**Aggregation type as listed in\r\nhttps://jarvis.corp.linkedin.com/codesearch/result/\r\n?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7\r\n*/aggregation:string/**feature name of the feature which would be computed.\r\nwe need feature name here for 2 main reasons.\r\n1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and\r\nwe want to leverage that.\r\n2. For default values. Similar to above, there are existing APIs which create default value map from feature name ->\r\ndefault value.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}}{namespace com.linkedin.feathr.compute/**Representation of a transformation node.*/record Transformation includes AbstractNode{/**An array of node references which should be considered as input to apply the transformation function.*/inputs:array[NodeReference]/**The transformation function.*/function:/**The transformation function*/record TransformationFunction{/**Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class.\r\n*/operator:OperatorId/**The various attributes required to represent the transformation function are captured in a map format.\r\nFor example, mvel expression or java udf class name*/parameters:optional map[string,string]}/**Feature name here is used so we retain feature name, type, and default values even after graph is resolved.\r\nFeature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}}{namespace com.linkedin.feathr.compute/**A temporary node which would exist only while parsing the graph. For example, when parsing an object if there is a reference to a feature\r\nname, we will create an external node. This would get resolved later in the computation.*/record External includes AbstractNode{/**Name of the external object it should refer to.*/name:string}}]]", SchemaFormatType.PDL)); + + public AnyNodeArray() { + this(new DataList()); + } + + public AnyNodeArray(int initialCapacity) { + this(new DataList(initialCapacity)); + } + + public AnyNodeArray(Collection c) { + this(new DataList(c.size())); + addAll(c); + } + + public AnyNodeArray(DataList data) { + super(data, SCHEMA, AnyNode.class); + } + + public AnyNodeArray(AnyNode first, AnyNode... rest) { + this(new DataList((rest.length + 1))); + add(first); + addAll(Arrays.asList(rest)); + } + + public static ArrayDataSchema dataSchema() { + return SCHEMA; + } + + public static AnyNodeArray.ProjectionMask createMask() { + return new AnyNodeArray.ProjectionMask(); + } + + @Override + public AnyNodeArray clone() + throws CloneNotSupportedException + { + AnyNodeArray __clone = ((AnyNodeArray) super.clone()); + return __clone; + } + + @Override + public AnyNodeArray copy() + throws CloneNotSupportedException + { + AnyNodeArray __copy = ((AnyNodeArray) super.copy()); + return __copy; + } + + @Override + protected AnyNode coerceOutput(Object object) + throws TemplateOutputCastException + { + assert(object != null); + return ((object == null)?null:new AnyNode(object)); + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.compute.AnyNode.Fields items() { + return new com.linkedin.feathr.compute.AnyNode.Fields(getPathComponents(), PathSpec.WILDCARD); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.AnyNode.ProjectionMask _itemsMask; + + ProjectionMask() { + super(4); + } + + public AnyNodeArray.ProjectionMask withItems(Function nestedMask) { + _itemsMask = nestedMask.apply(((_itemsMask == null)?AnyNode.createMask():_itemsMask)); + getDataMap().put("$*", _itemsMask.getDataMap()); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ComputeGraph.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ComputeGraph.java new file mode 100644 index 000000000..8258206ea --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ComputeGraph.java @@ -0,0 +1,464 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.IntegerMap; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * Feature computation graph. The passed in feature definition graph should get converted to this dependency graph. This graph is a + * direct translation of all the features present, and is not optimized with respect to the join config. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\ComputeGraph.pdl.") +public class ComputeGraph + extends RecordTemplate +{ + + private final static ComputeGraph.Fields _fields = new ComputeGraph.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Feature computation graph. The passed in feature definition graph should get converted to this dependency graph. This graph is a\r\ndirect translation of all the features present, and is not optimized with respect to the join config.*/record ComputeGraph{/**The nodes in the graph (order does not matter)*/nodes:array[/**A typeref for all the different types of nodes.*/typeref AnyNode=union[/**A node to represent an aggregation step. The aggregation inputs like the groupBy field, agg function are delegated to [[AggregationFunction]].\r\nThis node can represent a feature. As of now, in this step we will be using the SWA library from Spark-algorithms.*/record Aggregation includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node.*/input:/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}/**All the aggregation related parameters and functions are bundled into this.*/function:/**All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node.*/record AggregationFunction{/**The aggregation function.*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**All the aggregation parameters should be bundled into this map. For now, the possible parameters are:-\r\na. target_column - Aggregation column\r\nb. window_size - aggregation window size\r\nc. window unit - aggregation window unit (ex - day, hour)\r\nd. lateral_view_expression - definition of a lateral view for the feature.\r\ne. lateral_view_table_alias - An alias for the lateral view\r\nf. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression.\r\ng. groupBy - groupBy columns. Should be a sparkSql expression.*/parameters:optional map[string,string]}/**If the node is representing a feature, the feature name should be associated with the node.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}/**Representation of the datasource node. There are 3 types of datasource nodes:-\r\nContext - To represent the observation data entities (like the join key or passthrough feature columns)\r\nUpdate - To represent a non-timepartitioned datasource node.\r\nEvent - To represent a time-partitioned datasource node.\r\n\r\nTODO - Maybe, it makes sense more sense to refactor it by make this an abstract object, and deriving the three different nodes from it.*/record DataSource includes AbstractNode{/**Type of node, ie - Context, Update, Event*/sourceType:/**Type of datasource node.*/enum DataSourceType{/**Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log.*/UPDATE/**Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K\u00e2\u20ac\u2122d)\r\nover a limited window of time.*/EVENT/**Reprent the observation data entities (like the join key or passthrough feature columns)*/CONTEXT}/**for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI.*/externalSourceRef:string/**Raw key expression as entered by the user. This hocon parsing happens at the execution engine side.*/keyExpression:string/**mvel or spark or user-defined class*/keyExpressionType:/**Different key formats supported.\r\nTodo - We probably do not want to generalize this as a kind of key-operator in the core compute model,\r\nwith instances such as for MVEL or SQL being available (e.g. via an OperatorId reference).*/enum KeyExpressionType{/**Java-based MVEL*/MVEL/**Spark-SQL*/SQL/**Custom java/scala UDF*/UDF}/**File partition format.*/filePartitionFormat:optional string/**Timestamp column info, to be available only for an event datasource node.*/timestampColumnInfo:optional/**Representation of a timestamp column field*/record TimestampCol{/**Timestamp column expression.*/expression:string/**Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis*/format:string}}/**A node to represent a feature which is to be computed by using an already computed feature as the key.\r\nhttps://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?spaceKey=ENGS&title=feathr+Offline+User+Guide#FrameOfflineUserGuide-sequentialjoin*/record Lookup includes AbstractNode{/**An array of references to a node and keys.\r\n\r\nFor now, we do not support lookup of just a key reference, but we have added that as a placeholder.\r\n\r\nA node reference consists of node id and a key reference.\r\nIn sequential join the lookup key would be a combination of the\r\nfeature node representing the base feature (lookup node) and the key associated with it. For example,:-\r\nseqJoinFeature: {\r\n base: {key: x, feature: baseFeature}\r\n expansion: {key: y, feature: expansionFeature}\r\n aggregation: UNION\r\n}\r\nHere, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would\r\npoint to the index of \"x\" in the key array of baseFeature.*/lookupKey:array[union[NodeReference,KeyReference]]/**The node id of the node containing the expansion feature.*/lookupNode:NodeId/**Aggregation type as listed in\r\nhttps://jarvis.corp.linkedin.com/codesearch/result/\r\n?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7\r\n*/aggregation:string/**feature name of the feature which would be computed.\r\nwe need feature name here for 2 main reasons.\r\n1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and\r\nwe want to leverage that.\r\n2. For default values. Similar to above, there are existing APIs which create default value map from feature name ->\r\ndefault value.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}/**Representation of a transformation node.*/record Transformation includes AbstractNode{/**An array of node references which should be considered as input to apply the transformation function.*/inputs:array[NodeReference]/**The transformation function.*/function:/**The transformation function*/record TransformationFunction{/**Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class.\r\n*/operator:OperatorId/**The various attributes required to represent the transformation function are captured in a map format.\r\nFor example, mvel expression or java udf class name*/parameters:optional map[string,string]}/**Feature name here is used so we retain feature name, type, and default values even after graph is resolved.\r\nFeature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}/**A temporary node which would exist only while parsing the graph. For example, when parsing an object if there is a reference to a feature\r\nname, we will create an external node. This would get resolved later in the computation.*/record External includes AbstractNode{/**Name of the external object it should refer to.*/name:string}]]/**Map from feature name to node ID, for those nodes in the graph that represent named features.*/featureNames:map[string,int]}", SchemaFormatType.PDL)); + private AnyNodeArray _nodesField = null; + private IntegerMap _featureNamesField = null; + private ComputeGraph.ChangeListener __changeListener = new ComputeGraph.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Nodes = SCHEMA.getField("nodes"); + private final static RecordDataSchema.Field FIELD_FeatureNames = SCHEMA.getField("featureNames"); + + public ComputeGraph() { + super(new DataMap(3, 0.75F), SCHEMA, 3); + addChangeListener(__changeListener); + } + + public ComputeGraph(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static ComputeGraph.Fields fields() { + return _fields; + } + + public static ComputeGraph.ProjectionMask createMask() { + return new ComputeGraph.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for nodes + * + * @see ComputeGraph.Fields#nodes + */ + public boolean hasNodes() { + if (_nodesField!= null) { + return true; + } + return super._map.containsKey("nodes"); + } + + /** + * Remover for nodes + * + * @see ComputeGraph.Fields#nodes + */ + public void removeNodes() { + super._map.remove("nodes"); + } + + /** + * Getter for nodes + * + * @see ComputeGraph.Fields#nodes + */ + public AnyNodeArray getNodes(GetMode mode) { + switch (mode) { + case STRICT: + return getNodes(); + case DEFAULT: + case NULL: + if (_nodesField!= null) { + return _nodesField; + } else { + Object __rawValue = super._map.get("nodes"); + _nodesField = ((__rawValue == null)?null:new AnyNodeArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _nodesField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for nodes + * + * @return + * Required field. Could be null for partial record. + * @see ComputeGraph.Fields#nodes + */ + @Nonnull + public AnyNodeArray getNodes() { + if (_nodesField!= null) { + return _nodesField; + } else { + Object __rawValue = super._map.get("nodes"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("nodes"); + } + _nodesField = ((__rawValue == null)?null:new AnyNodeArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _nodesField; + } + } + + /** + * Setter for nodes + * + * @see ComputeGraph.Fields#nodes + */ + public ComputeGraph setNodes(AnyNodeArray value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setNodes(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field nodes of com.linkedin.feathr.compute.ComputeGraph"); + } else { + CheckedUtil.putWithoutChecking(super._map, "nodes", value.data()); + _nodesField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeNodes(); + } else { + CheckedUtil.putWithoutChecking(super._map, "nodes", value.data()); + _nodesField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "nodes", value.data()); + _nodesField = value; + } + break; + } + return this; + } + + /** + * Setter for nodes + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see ComputeGraph.Fields#nodes + */ + public ComputeGraph setNodes( + @Nonnull + AnyNodeArray value) { + if (value == null) { + throw new NullPointerException("Cannot set field nodes of com.linkedin.feathr.compute.ComputeGraph to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "nodes", value.data()); + _nodesField = value; + } + return this; + } + + /** + * Existence checker for featureNames + * + * @see ComputeGraph.Fields#featureNames + */ + public boolean hasFeatureNames() { + if (_featureNamesField!= null) { + return true; + } + return super._map.containsKey("featureNames"); + } + + /** + * Remover for featureNames + * + * @see ComputeGraph.Fields#featureNames + */ + public void removeFeatureNames() { + super._map.remove("featureNames"); + } + + /** + * Getter for featureNames + * + * @see ComputeGraph.Fields#featureNames + */ + public IntegerMap getFeatureNames(GetMode mode) { + switch (mode) { + case STRICT: + return getFeatureNames(); + case DEFAULT: + case NULL: + if (_featureNamesField!= null) { + return _featureNamesField; + } else { + Object __rawValue = super._map.get("featureNames"); + _featureNamesField = ((__rawValue == null)?null:new IntegerMap(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _featureNamesField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for featureNames + * + * @return + * Required field. Could be null for partial record. + * @see ComputeGraph.Fields#featureNames + */ + @Nonnull + public IntegerMap getFeatureNames() { + if (_featureNamesField!= null) { + return _featureNamesField; + } else { + Object __rawValue = super._map.get("featureNames"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("featureNames"); + } + _featureNamesField = ((__rawValue == null)?null:new IntegerMap(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _featureNamesField; + } + } + + /** + * Setter for featureNames + * + * @see ComputeGraph.Fields#featureNames + */ + public ComputeGraph setFeatureNames(IntegerMap value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFeatureNames(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field featureNames of com.linkedin.feathr.compute.ComputeGraph"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureNames", value.data()); + _featureNamesField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeFeatureNames(); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureNames", value.data()); + _featureNamesField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "featureNames", value.data()); + _featureNamesField = value; + } + break; + } + return this; + } + + /** + * Setter for featureNames + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see ComputeGraph.Fields#featureNames + */ + public ComputeGraph setFeatureNames( + @Nonnull + IntegerMap value) { + if (value == null) { + throw new NullPointerException("Cannot set field featureNames of com.linkedin.feathr.compute.ComputeGraph to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureNames", value.data()); + _featureNamesField = value; + } + return this; + } + + @Override + public ComputeGraph clone() + throws CloneNotSupportedException + { + ComputeGraph __clone = ((ComputeGraph) super.clone()); + __clone.__changeListener = new ComputeGraph.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public ComputeGraph copy() + throws CloneNotSupportedException + { + ComputeGraph __copy = ((ComputeGraph) super.copy()); + __copy._nodesField = null; + __copy._featureNamesField = null; + __copy.__changeListener = new ComputeGraph.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final ComputeGraph __objectRef; + + private ChangeListener(ComputeGraph reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "nodes": + __objectRef._nodesField = null; + break; + case "featureNames": + __objectRef._featureNamesField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The nodes in the graph (order does not matter) + * + */ + public com.linkedin.feathr.compute.AnyNodeArray.Fields nodes() { + return new com.linkedin.feathr.compute.AnyNodeArray.Fields(getPathComponents(), "nodes"); + } + + /** + * The nodes in the graph (order does not matter) + * + */ + public PathSpec nodes(Integer start, Integer count) { + PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "nodes"); + if (start!= null) { + arrayPathSpec.setAttribute("start", start); + } + if (count!= null) { + arrayPathSpec.setAttribute("count", count); + } + return arrayPathSpec; + } + + /** + * Map from feature name to node ID, for those nodes in the graph that represent named features. + * + */ + public PathSpec featureNames() { + return new PathSpec(getPathComponents(), "featureNames"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.AnyNodeArray.ProjectionMask _nodesMask; + + ProjectionMask() { + super(3); + } + + /** + * The nodes in the graph (order does not matter) + * + */ + public ComputeGraph.ProjectionMask withNodes(Function nestedMask) { + _nodesMask = nestedMask.apply(((_nodesMask == null)?AnyNodeArray.createMask():_nodesMask)); + getDataMap().put("nodes", _nodesMask.getDataMap()); + return this; + } + + /** + * The nodes in the graph (order does not matter) + * + */ + public ComputeGraph.ProjectionMask withNodes() { + _nodesMask = null; + getDataMap().put("nodes", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * The nodes in the graph (order does not matter) + * + */ + public ComputeGraph.ProjectionMask withNodes(Function nestedMask, Integer start, Integer count) { + _nodesMask = nestedMask.apply(((_nodesMask == null)?AnyNodeArray.createMask():_nodesMask)); + getDataMap().put("nodes", _nodesMask.getDataMap()); + if (start!= null) { + getDataMap().getDataMap("nodes").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("nodes").put("$count", count); + } + return this; + } + + /** + * The nodes in the graph (order does not matter) + * + */ + public ComputeGraph.ProjectionMask withNodes(Integer start, Integer count) { + _nodesMask = null; + getDataMap().put("nodes", new DataMap(3)); + if (start!= null) { + getDataMap().getDataMap("nodes").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("nodes").put("$count", count); + } + return this; + } + + /** + * Map from feature name to node ID, for those nodes in the graph that represent named features. + * + */ + public ComputeGraph.ProjectionMask withFeatureNames() { + getDataMap().put("featureNames", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ConcreteKey.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ConcreteKey.java new file mode 100644 index 000000000..f49dd9894 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ConcreteKey.java @@ -0,0 +1,300 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.IntegerArray; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * The key (node) for which the node in question is requested. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\ConcreteKey.pdl.") +public class ConcreteKey + extends RecordTemplate +{ + + private final static ConcreteKey.Fields _fields = new ConcreteKey.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[/**A type ref to int node id*/typeref NodeId=int]}", SchemaFormatType.PDL)); + private IntegerArray _keyField = null; + private ConcreteKey.ChangeListener __changeListener = new ConcreteKey.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Key = SCHEMA.getField("key"); + + public ConcreteKey() { + super(new DataMap(2, 0.75F), SCHEMA, 2); + addChangeListener(__changeListener); + } + + public ConcreteKey(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static ConcreteKey.Fields fields() { + return _fields; + } + + public static ConcreteKey.ProjectionMask createMask() { + return new ConcreteKey.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for key + * + * @see ConcreteKey.Fields#key + */ + public boolean hasKey() { + if (_keyField!= null) { + return true; + } + return super._map.containsKey("key"); + } + + /** + * Remover for key + * + * @see ConcreteKey.Fields#key + */ + public void removeKey() { + super._map.remove("key"); + } + + /** + * Getter for key + * + * @see ConcreteKey.Fields#key + */ + public IntegerArray getKey(GetMode mode) { + switch (mode) { + case STRICT: + return getKey(); + case DEFAULT: + case NULL: + if (_keyField!= null) { + return _keyField; + } else { + Object __rawValue = super._map.get("key"); + _keyField = ((__rawValue == null)?null:new IntegerArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _keyField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for key + * + * @return + * Required field. Could be null for partial record. + * @see ConcreteKey.Fields#key + */ + @Nonnull + public IntegerArray getKey() { + if (_keyField!= null) { + return _keyField; + } else { + Object __rawValue = super._map.get("key"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("key"); + } + _keyField = ((__rawValue == null)?null:new IntegerArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _keyField; + } + } + + /** + * Setter for key + * + * @see ConcreteKey.Fields#key + */ + public ConcreteKey setKey(IntegerArray value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setKey(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field key of com.linkedin.feathr.compute.ConcreteKey"); + } else { + CheckedUtil.putWithoutChecking(super._map, "key", value.data()); + _keyField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeKey(); + } else { + CheckedUtil.putWithoutChecking(super._map, "key", value.data()); + _keyField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "key", value.data()); + _keyField = value; + } + break; + } + return this; + } + + /** + * Setter for key + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see ConcreteKey.Fields#key + */ + public ConcreteKey setKey( + @Nonnull + IntegerArray value) { + if (value == null) { + throw new NullPointerException("Cannot set field key of com.linkedin.feathr.compute.ConcreteKey to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "key", value.data()); + _keyField = value; + } + return this; + } + + @Override + public ConcreteKey clone() + throws CloneNotSupportedException + { + ConcreteKey __clone = ((ConcreteKey) super.clone()); + __clone.__changeListener = new ConcreteKey.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public ConcreteKey copy() + throws CloneNotSupportedException + { + ConcreteKey __copy = ((ConcreteKey) super.copy()); + __copy._keyField = null; + __copy.__changeListener = new ConcreteKey.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final ConcreteKey __objectRef; + + private ChangeListener(ConcreteKey reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "key": + __objectRef._keyField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x. + * The main exception would be for a Lookup feature, in which case it would point to another node where the lookup + * key gets computed. + * + */ + public PathSpec key() { + return new PathSpec(getPathComponents(), "key"); + } + + /** + * Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x. + * The main exception would be for a Lookup feature, in which case it would point to another node where the lookup + * key gets computed. + * + */ + public PathSpec key(Integer start, Integer count) { + PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "key"); + if (start!= null) { + arrayPathSpec.setAttribute("start", start); + } + if (count!= null) { + arrayPathSpec.setAttribute("count", count); + } + return arrayPathSpec; + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(2); + } + + /** + * Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x. + * The main exception would be for a Lookup feature, in which case it would point to another node where the lookup + * key gets computed. + * + */ + public ConcreteKey.ProjectionMask withKey() { + getDataMap().put("key", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x. + * The main exception would be for a Lookup feature, in which case it would point to another node where the lookup + * key gets computed. + * + */ + public ConcreteKey.ProjectionMask withKey(Integer start, Integer count) { + getDataMap().put("key", new DataMap(3)); + if (start!= null) { + getDataMap().getDataMap("key").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("key").put("$count", count); + } + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSource.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSource.java new file mode 100644 index 000000000..21b44aa26 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSource.java @@ -0,0 +1,1232 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * Representation of the datasource node. There are 3 types of datasource nodes:- + * Context - To represent the observation data entities (like the join key or passthrough feature columns) + * Update - To represent a non-timepartitioned datasource node. + * Event - To represent a time-partitioned datasource node. + * + * TODO - Maybe, it makes sense more sense to refactor it by make this an abstract object, and deriving the three different nodes from it. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\DataSource.pdl.") +public class DataSource + extends RecordTemplate +{ + + private final static DataSource.Fields _fields = new DataSource.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Representation of the datasource node. There are 3 types of datasource nodes:-\r\nContext - To represent the observation data entities (like the join key or passthrough feature columns)\r\nUpdate - To represent a non-timepartitioned datasource node.\r\nEvent - To represent a time-partitioned datasource node.\r\n\r\nTODO - Maybe, it makes sense more sense to refactor it by make this an abstract object, and deriving the three different nodes from it.*/record DataSource includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**Type of node, ie - Context, Update, Event*/sourceType:/**Type of datasource node.*/enum DataSourceType{/**Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log.*/UPDATE/**Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K\u00e2\u20ac\u2122d)\r\nover a limited window of time.*/EVENT/**Reprent the observation data entities (like the join key or passthrough feature columns)*/CONTEXT}/**for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI.*/externalSourceRef:string/**Raw key expression as entered by the user. This hocon parsing happens at the execution engine side.*/keyExpression:string/**mvel or spark or user-defined class*/keyExpressionType:/**Different key formats supported.\r\nTodo - We probably do not want to generalize this as a kind of key-operator in the core compute model,\r\nwith instances such as for MVEL or SQL being available (e.g. via an OperatorId reference).*/enum KeyExpressionType{/**Java-based MVEL*/MVEL/**Spark-SQL*/SQL/**Custom java/scala UDF*/UDF}/**File partition format.*/filePartitionFormat:optional string/**Timestamp column info, to be available only for an event datasource node.*/timestampColumnInfo:optional/**Representation of a timestamp column field*/record TimestampCol{/**Timestamp column expression.*/expression:string/**Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis*/format:string}}", SchemaFormatType.PDL)); + private Integer _idField = null; + private ConcreteKey _concreteKeyField = null; + private DataSourceType _sourceTypeField = null; + private String _externalSourceRefField = null; + private String _keyExpressionField = null; + private KeyExpressionType _keyExpressionTypeField = null; + private String _filePartitionFormatField = null; + private TimestampCol _timestampColumnInfoField = null; + private DataSource.ChangeListener __changeListener = new DataSource.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); + private final static RecordDataSchema.Field FIELD_ConcreteKey = SCHEMA.getField("concreteKey"); + private final static RecordDataSchema.Field FIELD_SourceType = SCHEMA.getField("sourceType"); + private final static RecordDataSchema.Field FIELD_ExternalSourceRef = SCHEMA.getField("externalSourceRef"); + private final static RecordDataSchema.Field FIELD_KeyExpression = SCHEMA.getField("keyExpression"); + private final static RecordDataSchema.Field FIELD_KeyExpressionType = SCHEMA.getField("keyExpressionType"); + private final static RecordDataSchema.Field FIELD_FilePartitionFormat = SCHEMA.getField("filePartitionFormat"); + private final static RecordDataSchema.Field FIELD_TimestampColumnInfo = SCHEMA.getField("timestampColumnInfo"); + + public DataSource() { + super(new DataMap(11, 0.75F), SCHEMA, 3); + addChangeListener(__changeListener); + } + + public DataSource(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static DataSource.Fields fields() { + return _fields; + } + + public static DataSource.ProjectionMask createMask() { + return new DataSource.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for id + * + * @see DataSource.Fields#id + */ + public boolean hasId() { + if (_idField!= null) { + return true; + } + return super._map.containsKey("id"); + } + + /** + * Remover for id + * + * @see DataSource.Fields#id + */ + public void removeId() { + super._map.remove("id"); + } + + /** + * Getter for id + * + * @see DataSource.Fields#id + */ + public Integer getId(GetMode mode) { + switch (mode) { + case STRICT: + return getId(); + case DEFAULT: + case NULL: + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for id + * + * @return + * Required field. Could be null for partial record. + * @see DataSource.Fields#id + */ + @Nonnull + public Integer getId() { + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("id"); + } + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + + /** + * Setter for id + * + * @see DataSource.Fields#id + */ + public DataSource setId(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setId(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.DataSource"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeId(); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + } + return this; + } + + /** + * Setter for id + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see DataSource.Fields#id + */ + public DataSource setId( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.DataSource to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + return this; + } + + /** + * Setter for id + * + * @see DataSource.Fields#id + */ + public DataSource setId(int value) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + return this; + } + + /** + * Existence checker for concreteKey + * + * @see DataSource.Fields#concreteKey + */ + public boolean hasConcreteKey() { + if (_concreteKeyField!= null) { + return true; + } + return super._map.containsKey("concreteKey"); + } + + /** + * Remover for concreteKey + * + * @see DataSource.Fields#concreteKey + */ + public void removeConcreteKey() { + super._map.remove("concreteKey"); + } + + /** + * Getter for concreteKey + * + * @see DataSource.Fields#concreteKey + */ + public ConcreteKey getConcreteKey(GetMode mode) { + return getConcreteKey(); + } + + /** + * Getter for concreteKey + * + * @return + * Optional field. Always check for null. + * @see DataSource.Fields#concreteKey + */ + @Nullable + public ConcreteKey getConcreteKey() { + if (_concreteKeyField!= null) { + return _concreteKeyField; + } else { + Object __rawValue = super._map.get("concreteKey"); + _concreteKeyField = ((__rawValue == null)?null:new ConcreteKey(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _concreteKeyField; + } + } + + /** + * Setter for concreteKey + * + * @see DataSource.Fields#concreteKey + */ + public DataSource setConcreteKey(ConcreteKey value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setConcreteKey(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeConcreteKey(); + } else { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + break; + } + return this; + } + + /** + * Setter for concreteKey + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see DataSource.Fields#concreteKey + */ + public DataSource setConcreteKey( + @Nonnull + ConcreteKey value) { + if (value == null) { + throw new NullPointerException("Cannot set field concreteKey of com.linkedin.feathr.compute.DataSource to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + return this; + } + + /** + * Existence checker for sourceType + * + * @see DataSource.Fields#sourceType + */ + public boolean hasSourceType() { + if (_sourceTypeField!= null) { + return true; + } + return super._map.containsKey("sourceType"); + } + + /** + * Remover for sourceType + * + * @see DataSource.Fields#sourceType + */ + public void removeSourceType() { + super._map.remove("sourceType"); + } + + /** + * Getter for sourceType + * + * @see DataSource.Fields#sourceType + */ + public DataSourceType getSourceType(GetMode mode) { + switch (mode) { + case STRICT: + return getSourceType(); + case DEFAULT: + case NULL: + if (_sourceTypeField!= null) { + return _sourceTypeField; + } else { + Object __rawValue = super._map.get("sourceType"); + _sourceTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, DataSourceType.class, DataSourceType.$UNKNOWN); + return _sourceTypeField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for sourceType + * + * @return + * Required field. Could be null for partial record. + * @see DataSource.Fields#sourceType + */ + @Nonnull + public DataSourceType getSourceType() { + if (_sourceTypeField!= null) { + return _sourceTypeField; + } else { + Object __rawValue = super._map.get("sourceType"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("sourceType"); + } + _sourceTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, DataSourceType.class, DataSourceType.$UNKNOWN); + return _sourceTypeField; + } + } + + /** + * Setter for sourceType + * + * @see DataSource.Fields#sourceType + */ + public DataSource setSourceType(DataSourceType value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setSourceType(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field sourceType of com.linkedin.feathr.compute.DataSource"); + } else { + CheckedUtil.putWithoutChecking(super._map, "sourceType", value.name()); + _sourceTypeField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeSourceType(); + } else { + CheckedUtil.putWithoutChecking(super._map, "sourceType", value.name()); + _sourceTypeField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "sourceType", value.name()); + _sourceTypeField = value; + } + break; + } + return this; + } + + /** + * Setter for sourceType + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see DataSource.Fields#sourceType + */ + public DataSource setSourceType( + @Nonnull + DataSourceType value) { + if (value == null) { + throw new NullPointerException("Cannot set field sourceType of com.linkedin.feathr.compute.DataSource to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "sourceType", value.name()); + _sourceTypeField = value; + } + return this; + } + + /** + * Existence checker for externalSourceRef + * + * @see DataSource.Fields#externalSourceRef + */ + public boolean hasExternalSourceRef() { + if (_externalSourceRefField!= null) { + return true; + } + return super._map.containsKey("externalSourceRef"); + } + + /** + * Remover for externalSourceRef + * + * @see DataSource.Fields#externalSourceRef + */ + public void removeExternalSourceRef() { + super._map.remove("externalSourceRef"); + } + + /** + * Getter for externalSourceRef + * + * @see DataSource.Fields#externalSourceRef + */ + public String getExternalSourceRef(GetMode mode) { + switch (mode) { + case STRICT: + return getExternalSourceRef(); + case DEFAULT: + case NULL: + if (_externalSourceRefField!= null) { + return _externalSourceRefField; + } else { + Object __rawValue = super._map.get("externalSourceRef"); + _externalSourceRefField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _externalSourceRefField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for externalSourceRef + * + * @return + * Required field. Could be null for partial record. + * @see DataSource.Fields#externalSourceRef + */ + @Nonnull + public String getExternalSourceRef() { + if (_externalSourceRefField!= null) { + return _externalSourceRefField; + } else { + Object __rawValue = super._map.get("externalSourceRef"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("externalSourceRef"); + } + _externalSourceRefField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _externalSourceRefField; + } + } + + /** + * Setter for externalSourceRef + * + * @see DataSource.Fields#externalSourceRef + */ + public DataSource setExternalSourceRef(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setExternalSourceRef(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field externalSourceRef of com.linkedin.feathr.compute.DataSource"); + } else { + CheckedUtil.putWithoutChecking(super._map, "externalSourceRef", value); + _externalSourceRefField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeExternalSourceRef(); + } else { + CheckedUtil.putWithoutChecking(super._map, "externalSourceRef", value); + _externalSourceRefField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "externalSourceRef", value); + _externalSourceRefField = value; + } + break; + } + return this; + } + + /** + * Setter for externalSourceRef + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see DataSource.Fields#externalSourceRef + */ + public DataSource setExternalSourceRef( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field externalSourceRef of com.linkedin.feathr.compute.DataSource to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "externalSourceRef", value); + _externalSourceRefField = value; + } + return this; + } + + /** + * Existence checker for keyExpression + * + * @see DataSource.Fields#keyExpression + */ + public boolean hasKeyExpression() { + if (_keyExpressionField!= null) { + return true; + } + return super._map.containsKey("keyExpression"); + } + + /** + * Remover for keyExpression + * + * @see DataSource.Fields#keyExpression + */ + public void removeKeyExpression() { + super._map.remove("keyExpression"); + } + + /** + * Getter for keyExpression + * + * @see DataSource.Fields#keyExpression + */ + public String getKeyExpression(GetMode mode) { + switch (mode) { + case STRICT: + return getKeyExpression(); + case DEFAULT: + case NULL: + if (_keyExpressionField!= null) { + return _keyExpressionField; + } else { + Object __rawValue = super._map.get("keyExpression"); + _keyExpressionField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _keyExpressionField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for keyExpression + * + * @return + * Required field. Could be null for partial record. + * @see DataSource.Fields#keyExpression + */ + @Nonnull + public String getKeyExpression() { + if (_keyExpressionField!= null) { + return _keyExpressionField; + } else { + Object __rawValue = super._map.get("keyExpression"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("keyExpression"); + } + _keyExpressionField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _keyExpressionField; + } + } + + /** + * Setter for keyExpression + * + * @see DataSource.Fields#keyExpression + */ + public DataSource setKeyExpression(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setKeyExpression(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field keyExpression of com.linkedin.feathr.compute.DataSource"); + } else { + CheckedUtil.putWithoutChecking(super._map, "keyExpression", value); + _keyExpressionField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeKeyExpression(); + } else { + CheckedUtil.putWithoutChecking(super._map, "keyExpression", value); + _keyExpressionField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "keyExpression", value); + _keyExpressionField = value; + } + break; + } + return this; + } + + /** + * Setter for keyExpression + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see DataSource.Fields#keyExpression + */ + public DataSource setKeyExpression( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field keyExpression of com.linkedin.feathr.compute.DataSource to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "keyExpression", value); + _keyExpressionField = value; + } + return this; + } + + /** + * Existence checker for keyExpressionType + * + * @see DataSource.Fields#keyExpressionType + */ + public boolean hasKeyExpressionType() { + if (_keyExpressionTypeField!= null) { + return true; + } + return super._map.containsKey("keyExpressionType"); + } + + /** + * Remover for keyExpressionType + * + * @see DataSource.Fields#keyExpressionType + */ + public void removeKeyExpressionType() { + super._map.remove("keyExpressionType"); + } + + /** + * Getter for keyExpressionType + * + * @see DataSource.Fields#keyExpressionType + */ + public KeyExpressionType getKeyExpressionType(GetMode mode) { + switch (mode) { + case STRICT: + return getKeyExpressionType(); + case DEFAULT: + case NULL: + if (_keyExpressionTypeField!= null) { + return _keyExpressionTypeField; + } else { + Object __rawValue = super._map.get("keyExpressionType"); + _keyExpressionTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, KeyExpressionType.class, KeyExpressionType.$UNKNOWN); + return _keyExpressionTypeField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for keyExpressionType + * + * @return + * Required field. Could be null for partial record. + * @see DataSource.Fields#keyExpressionType + */ + @Nonnull + public KeyExpressionType getKeyExpressionType() { + if (_keyExpressionTypeField!= null) { + return _keyExpressionTypeField; + } else { + Object __rawValue = super._map.get("keyExpressionType"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("keyExpressionType"); + } + _keyExpressionTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, KeyExpressionType.class, KeyExpressionType.$UNKNOWN); + return _keyExpressionTypeField; + } + } + + /** + * Setter for keyExpressionType + * + * @see DataSource.Fields#keyExpressionType + */ + public DataSource setKeyExpressionType(KeyExpressionType value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setKeyExpressionType(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field keyExpressionType of com.linkedin.feathr.compute.DataSource"); + } else { + CheckedUtil.putWithoutChecking(super._map, "keyExpressionType", value.name()); + _keyExpressionTypeField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeKeyExpressionType(); + } else { + CheckedUtil.putWithoutChecking(super._map, "keyExpressionType", value.name()); + _keyExpressionTypeField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "keyExpressionType", value.name()); + _keyExpressionTypeField = value; + } + break; + } + return this; + } + + /** + * Setter for keyExpressionType + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see DataSource.Fields#keyExpressionType + */ + public DataSource setKeyExpressionType( + @Nonnull + KeyExpressionType value) { + if (value == null) { + throw new NullPointerException("Cannot set field keyExpressionType of com.linkedin.feathr.compute.DataSource to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "keyExpressionType", value.name()); + _keyExpressionTypeField = value; + } + return this; + } + + /** + * Existence checker for filePartitionFormat + * + * @see DataSource.Fields#filePartitionFormat + */ + public boolean hasFilePartitionFormat() { + if (_filePartitionFormatField!= null) { + return true; + } + return super._map.containsKey("filePartitionFormat"); + } + + /** + * Remover for filePartitionFormat + * + * @see DataSource.Fields#filePartitionFormat + */ + public void removeFilePartitionFormat() { + super._map.remove("filePartitionFormat"); + } + + /** + * Getter for filePartitionFormat + * + * @see DataSource.Fields#filePartitionFormat + */ + public String getFilePartitionFormat(GetMode mode) { + return getFilePartitionFormat(); + } + + /** + * Getter for filePartitionFormat + * + * @return + * Optional field. Always check for null. + * @see DataSource.Fields#filePartitionFormat + */ + @Nullable + public String getFilePartitionFormat() { + if (_filePartitionFormatField!= null) { + return _filePartitionFormatField; + } else { + Object __rawValue = super._map.get("filePartitionFormat"); + _filePartitionFormatField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _filePartitionFormatField; + } + } + + /** + * Setter for filePartitionFormat + * + * @see DataSource.Fields#filePartitionFormat + */ + public DataSource setFilePartitionFormat(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFilePartitionFormat(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeFilePartitionFormat(); + } else { + CheckedUtil.putWithoutChecking(super._map, "filePartitionFormat", value); + _filePartitionFormatField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "filePartitionFormat", value); + _filePartitionFormatField = value; + } + break; + } + return this; + } + + /** + * Setter for filePartitionFormat + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see DataSource.Fields#filePartitionFormat + */ + public DataSource setFilePartitionFormat( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field filePartitionFormat of com.linkedin.feathr.compute.DataSource to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "filePartitionFormat", value); + _filePartitionFormatField = value; + } + return this; + } + + /** + * Existence checker for timestampColumnInfo + * + * @see DataSource.Fields#timestampColumnInfo + */ + public boolean hasTimestampColumnInfo() { + if (_timestampColumnInfoField!= null) { + return true; + } + return super._map.containsKey("timestampColumnInfo"); + } + + /** + * Remover for timestampColumnInfo + * + * @see DataSource.Fields#timestampColumnInfo + */ + public void removeTimestampColumnInfo() { + super._map.remove("timestampColumnInfo"); + } + + /** + * Getter for timestampColumnInfo + * + * @see DataSource.Fields#timestampColumnInfo + */ + public TimestampCol getTimestampColumnInfo(GetMode mode) { + return getTimestampColumnInfo(); + } + + /** + * Getter for timestampColumnInfo + * + * @return + * Optional field. Always check for null. + * @see DataSource.Fields#timestampColumnInfo + */ + @Nullable + public TimestampCol getTimestampColumnInfo() { + if (_timestampColumnInfoField!= null) { + return _timestampColumnInfoField; + } else { + Object __rawValue = super._map.get("timestampColumnInfo"); + _timestampColumnInfoField = ((__rawValue == null)?null:new TimestampCol(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _timestampColumnInfoField; + } + } + + /** + * Setter for timestampColumnInfo + * + * @see DataSource.Fields#timestampColumnInfo + */ + public DataSource setTimestampColumnInfo(TimestampCol value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setTimestampColumnInfo(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeTimestampColumnInfo(); + } else { + CheckedUtil.putWithoutChecking(super._map, "timestampColumnInfo", value.data()); + _timestampColumnInfoField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "timestampColumnInfo", value.data()); + _timestampColumnInfoField = value; + } + break; + } + return this; + } + + /** + * Setter for timestampColumnInfo + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see DataSource.Fields#timestampColumnInfo + */ + public DataSource setTimestampColumnInfo( + @Nonnull + TimestampCol value) { + if (value == null) { + throw new NullPointerException("Cannot set field timestampColumnInfo of com.linkedin.feathr.compute.DataSource to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "timestampColumnInfo", value.data()); + _timestampColumnInfoField = value; + } + return this; + } + + @Override + public DataSource clone() + throws CloneNotSupportedException + { + DataSource __clone = ((DataSource) super.clone()); + __clone.__changeListener = new DataSource.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public DataSource copy() + throws CloneNotSupportedException + { + DataSource __copy = ((DataSource) super.copy()); + __copy._keyExpressionTypeField = null; + __copy._sourceTypeField = null; + __copy._externalSourceRefField = null; + __copy._timestampColumnInfoField = null; + __copy._keyExpressionField = null; + __copy._filePartitionFormatField = null; + __copy._idField = null; + __copy._concreteKeyField = null; + __copy.__changeListener = new DataSource.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final DataSource __objectRef; + + private ChangeListener(DataSource reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "keyExpressionType": + __objectRef._keyExpressionTypeField = null; + break; + case "sourceType": + __objectRef._sourceTypeField = null; + break; + case "externalSourceRef": + __objectRef._externalSourceRefField = null; + break; + case "timestampColumnInfo": + __objectRef._timestampColumnInfoField = null; + break; + case "keyExpression": + __objectRef._keyExpressionField = null; + break; + case "filePartitionFormat": + __objectRef._filePartitionFormatField = null; + break; + case "id": + __objectRef._idField = null; + break; + case "concreteKey": + __objectRef._concreteKeyField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The node would be represented by this id. + * + */ + public PathSpec id() { + return new PathSpec(getPathComponents(), "id"); + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public com.linkedin.feathr.compute.ConcreteKey.Fields concreteKey() { + return new com.linkedin.feathr.compute.ConcreteKey.Fields(getPathComponents(), "concreteKey"); + } + + /** + * Type of node, ie - Context, Update, Event + * + */ + public PathSpec sourceType() { + return new PathSpec(getPathComponents(), "sourceType"); + } + + /** + * for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI. + * + */ + public PathSpec externalSourceRef() { + return new PathSpec(getPathComponents(), "externalSourceRef"); + } + + /** + * Raw key expression as entered by the user. This hocon parsing happens at the execution engine side. + * + */ + public PathSpec keyExpression() { + return new PathSpec(getPathComponents(), "keyExpression"); + } + + /** + * mvel or spark or user-defined class + * + */ + public PathSpec keyExpressionType() { + return new PathSpec(getPathComponents(), "keyExpressionType"); + } + + /** + * File partition format. + * + */ + public PathSpec filePartitionFormat() { + return new PathSpec(getPathComponents(), "filePartitionFormat"); + } + + /** + * Timestamp column info, to be available only for an event datasource node. + * + */ + public com.linkedin.feathr.compute.TimestampCol.Fields timestampColumnInfo() { + return new com.linkedin.feathr.compute.TimestampCol.Fields(getPathComponents(), "timestampColumnInfo"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.ConcreteKey.ProjectionMask _concreteKeyMask; + private com.linkedin.feathr.compute.TimestampCol.ProjectionMask _timestampColumnInfoMask; + + ProjectionMask() { + super(11); + } + + /** + * The node would be represented by this id. + * + */ + public DataSource.ProjectionMask withId() { + getDataMap().put("id", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public DataSource.ProjectionMask withConcreteKey(Function nestedMask) { + _concreteKeyMask = nestedMask.apply(((_concreteKeyMask == null)?ConcreteKey.createMask():_concreteKeyMask)); + getDataMap().put("concreteKey", _concreteKeyMask.getDataMap()); + return this; + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public DataSource.ProjectionMask withConcreteKey() { + _concreteKeyMask = null; + getDataMap().put("concreteKey", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Type of node, ie - Context, Update, Event + * + */ + public DataSource.ProjectionMask withSourceType() { + getDataMap().put("sourceType", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI. + * + */ + public DataSource.ProjectionMask withExternalSourceRef() { + getDataMap().put("externalSourceRef", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Raw key expression as entered by the user. This hocon parsing happens at the execution engine side. + * + */ + public DataSource.ProjectionMask withKeyExpression() { + getDataMap().put("keyExpression", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * mvel or spark or user-defined class + * + */ + public DataSource.ProjectionMask withKeyExpressionType() { + getDataMap().put("keyExpressionType", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * File partition format. + * + */ + public DataSource.ProjectionMask withFilePartitionFormat() { + getDataMap().put("filePartitionFormat", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Timestamp column info, to be available only for an event datasource node. + * + */ + public DataSource.ProjectionMask withTimestampColumnInfo(Function nestedMask) { + _timestampColumnInfoMask = nestedMask.apply(((_timestampColumnInfoMask == null)?TimestampCol.createMask():_timestampColumnInfoMask)); + getDataMap().put("timestampColumnInfo", _timestampColumnInfoMask.getDataMap()); + return this; + } + + /** + * Timestamp column info, to be available only for an event datasource node. + * + */ + public DataSource.ProjectionMask withTimestampColumnInfo() { + _timestampColumnInfoMask = null; + getDataMap().put("timestampColumnInfo", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSourceType.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSourceType.java new file mode 100644 index 000000000..01d80c669 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSourceType.java @@ -0,0 +1,43 @@ + +package com.linkedin.feathr.compute; + +import javax.annotation.Generated; +import com.linkedin.data.schema.EnumDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; + + +/** + * Type of datasource node. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\DataSourceType.pdl.") +public enum DataSourceType { + + + /** + * Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log. + * + */ + UPDATE, + + /** + * Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K’d) + * over a limited window of time. + * + */ + EVENT, + + /** + * Reprent the observation data entities (like the join key or passthrough feature columns) + * + */ + CONTEXT, + $UNKNOWN; + private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Type of datasource node.*/enum DataSourceType{/**Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log.*/UPDATE/**Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K\u00e2\u20ac\u2122d)\r\nover a limited window of time.*/EVENT/**Reprent the observation data entities (like the join key or passthrough feature columns)*/CONTEXT}", SchemaFormatType.PDL)); + + public static EnumDataSchema dataSchema() { + return SCHEMA; + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DateTimeInterval.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DateTimeInterval.java new file mode 100644 index 000000000..016e9d3aa --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DateTimeInterval.java @@ -0,0 +1,377 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.SetMode; + + +/** + * Represent a data time interval + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\DateTimeInterval.pdl.") +public class DateTimeInterval + extends RecordTemplate +{ + + private final static DateTimeInterval.Fields _fields = new DateTimeInterval.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Represent a data time interval*/record DateTimeInterval{/**Represents the inclusive (greater than or equal to) value in which to start the range. This field is optional. An unset field here indicates an open range; for example, if end is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and start is not set, it would indicate times up to, but excluding, 1455309628000. Note that this interpretation was not originally documented. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact.*/start:optional/**Number of milliseconds since midnight, January 1, 1970 UTC. It must be a positive number*/@compliance=\"NONE\"typeref Time=long/**Represents the exclusive (strictly less than) value in which to end the range. This field is optional. An unset field here indicates an open range; for example, if start is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and end is not set, it would mean everything at, or after, 1455309628000. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact.*/end:optional Time}", SchemaFormatType.PDL)); + private Long _startField = null; + private Long _endField = null; + private DateTimeInterval.ChangeListener __changeListener = new DateTimeInterval.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Start = SCHEMA.getField("start"); + private final static RecordDataSchema.Field FIELD_End = SCHEMA.getField("end"); + + public DateTimeInterval() { + super(new DataMap(3, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public DateTimeInterval(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static DateTimeInterval.Fields fields() { + return _fields; + } + + public static DateTimeInterval.ProjectionMask createMask() { + return new DateTimeInterval.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for start + * + * @see DateTimeInterval.Fields#start + */ + public boolean hasStart() { + if (_startField!= null) { + return true; + } + return super._map.containsKey("start"); + } + + /** + * Remover for start + * + * @see DateTimeInterval.Fields#start + */ + public void removeStart() { + super._map.remove("start"); + } + + /** + * Getter for start + * + * @see DateTimeInterval.Fields#start + */ + public Long getStart(GetMode mode) { + return getStart(); + } + + /** + * Getter for start + * + * @return + * Optional field. Always check for null. + * @see DateTimeInterval.Fields#start + */ + @Nullable + public Long getStart() { + if (_startField!= null) { + return _startField; + } else { + Object __rawValue = super._map.get("start"); + _startField = DataTemplateUtil.coerceLongOutput(__rawValue); + return _startField; + } + } + + /** + * Setter for start + * + * @see DateTimeInterval.Fields#start + */ + public DateTimeInterval setStart(Long value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setStart(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeStart(); + } else { + CheckedUtil.putWithoutChecking(super._map, "start", DataTemplateUtil.coerceLongInput(value)); + _startField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "start", DataTemplateUtil.coerceLongInput(value)); + _startField = value; + } + break; + } + return this; + } + + /** + * Setter for start + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see DateTimeInterval.Fields#start + */ + public DateTimeInterval setStart( + @Nonnull + Long value) { + if (value == null) { + throw new NullPointerException("Cannot set field start of com.linkedin.feathr.compute.DateTimeInterval to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "start", DataTemplateUtil.coerceLongInput(value)); + _startField = value; + } + return this; + } + + /** + * Setter for start + * + * @see DateTimeInterval.Fields#start + */ + public DateTimeInterval setStart(long value) { + CheckedUtil.putWithoutChecking(super._map, "start", DataTemplateUtil.coerceLongInput(value)); + _startField = value; + return this; + } + + /** + * Existence checker for end + * + * @see DateTimeInterval.Fields#end + */ + public boolean hasEnd() { + if (_endField!= null) { + return true; + } + return super._map.containsKey("end"); + } + + /** + * Remover for end + * + * @see DateTimeInterval.Fields#end + */ + public void removeEnd() { + super._map.remove("end"); + } + + /** + * Getter for end + * + * @see DateTimeInterval.Fields#end + */ + public Long getEnd(GetMode mode) { + return getEnd(); + } + + /** + * Getter for end + * + * @return + * Optional field. Always check for null. + * @see DateTimeInterval.Fields#end + */ + @Nullable + public Long getEnd() { + if (_endField!= null) { + return _endField; + } else { + Object __rawValue = super._map.get("end"); + _endField = DataTemplateUtil.coerceLongOutput(__rawValue); + return _endField; + } + } + + /** + * Setter for end + * + * @see DateTimeInterval.Fields#end + */ + public DateTimeInterval setEnd(Long value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setEnd(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeEnd(); + } else { + CheckedUtil.putWithoutChecking(super._map, "end", DataTemplateUtil.coerceLongInput(value)); + _endField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "end", DataTemplateUtil.coerceLongInput(value)); + _endField = value; + } + break; + } + return this; + } + + /** + * Setter for end + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see DateTimeInterval.Fields#end + */ + public DateTimeInterval setEnd( + @Nonnull + Long value) { + if (value == null) { + throw new NullPointerException("Cannot set field end of com.linkedin.feathr.compute.DateTimeInterval to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "end", DataTemplateUtil.coerceLongInput(value)); + _endField = value; + } + return this; + } + + /** + * Setter for end + * + * @see DateTimeInterval.Fields#end + */ + public DateTimeInterval setEnd(long value) { + CheckedUtil.putWithoutChecking(super._map, "end", DataTemplateUtil.coerceLongInput(value)); + _endField = value; + return this; + } + + @Override + public DateTimeInterval clone() + throws CloneNotSupportedException + { + DateTimeInterval __clone = ((DateTimeInterval) super.clone()); + __clone.__changeListener = new DateTimeInterval.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public DateTimeInterval copy() + throws CloneNotSupportedException + { + DateTimeInterval __copy = ((DateTimeInterval) super.copy()); + __copy._startField = null; + __copy._endField = null; + __copy.__changeListener = new DateTimeInterval.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final DateTimeInterval __objectRef; + + private ChangeListener(DateTimeInterval reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "start": + __objectRef._startField = null; + break; + case "end": + __objectRef._endField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Represents the inclusive (greater than or equal to) value in which to start the range. This field is optional. An unset field here indicates an open range; for example, if end is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and start is not set, it would indicate times up to, but excluding, 1455309628000. Note that this interpretation was not originally documented. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact. + * + */ + public PathSpec start() { + return new PathSpec(getPathComponents(), "start"); + } + + /** + * Represents the exclusive (strictly less than) value in which to end the range. This field is optional. An unset field here indicates an open range; for example, if start is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and end is not set, it would mean everything at, or after, 1455309628000. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact. + * + */ + public PathSpec end() { + return new PathSpec(getPathComponents(), "end"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(3); + } + + /** + * Represents the inclusive (greater than or equal to) value in which to start the range. This field is optional. An unset field here indicates an open range; for example, if end is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and start is not set, it would indicate times up to, but excluding, 1455309628000. Note that this interpretation was not originally documented. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact. + * + */ + public DateTimeInterval.ProjectionMask withStart() { + getDataMap().put("start", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Represents the exclusive (strictly less than) value in which to end the range. This field is optional. An unset field here indicates an open range; for example, if start is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and end is not set, it would mean everything at, or after, 1455309628000. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact. + * + */ + public DateTimeInterval.ProjectionMask withEnd() { + getDataMap().put("end", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Dimension.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Dimension.java new file mode 100644 index 000000000..6712e608c --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Dimension.java @@ -0,0 +1,390 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Dimension.pdl.") +public class Dimension + extends RecordTemplate +{ + + private final static Dimension.Fields _fields = new Dimension.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}", SchemaFormatType.PDL)); + private DimensionType _typeField = null; + private Integer _shapeField = null; + private Dimension.ChangeListener __changeListener = new Dimension.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Type = SCHEMA.getField("type"); + private final static RecordDataSchema.Field FIELD_Shape = SCHEMA.getField("shape"); + + public Dimension() { + super(new DataMap(3, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public Dimension(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static Dimension.Fields fields() { + return _fields; + } + + public static Dimension.ProjectionMask createMask() { + return new Dimension.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for type + * + * @see Dimension.Fields#type + */ + public boolean hasType() { + if (_typeField!= null) { + return true; + } + return super._map.containsKey("type"); + } + + /** + * Remover for type + * + * @see Dimension.Fields#type + */ + public void removeType() { + super._map.remove("type"); + } + + /** + * Getter for type + * + * @see Dimension.Fields#type + */ + public DimensionType getType(GetMode mode) { + switch (mode) { + case STRICT: + return getType(); + case DEFAULT: + case NULL: + if (_typeField!= null) { + return _typeField; + } else { + Object __rawValue = super._map.get("type"); + _typeField = DataTemplateUtil.coerceEnumOutput(__rawValue, DimensionType.class, DimensionType.$UNKNOWN); + return _typeField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for type + * + * @return + * Required field. Could be null for partial record. + * @see Dimension.Fields#type + */ + @Nonnull + public DimensionType getType() { + if (_typeField!= null) { + return _typeField; + } else { + Object __rawValue = super._map.get("type"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("type"); + } + _typeField = DataTemplateUtil.coerceEnumOutput(__rawValue, DimensionType.class, DimensionType.$UNKNOWN); + return _typeField; + } + } + + /** + * Setter for type + * + * @see Dimension.Fields#type + */ + public Dimension setType(DimensionType value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setType(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field type of com.linkedin.feathr.compute.Dimension"); + } else { + CheckedUtil.putWithoutChecking(super._map, "type", value.name()); + _typeField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeType(); + } else { + CheckedUtil.putWithoutChecking(super._map, "type", value.name()); + _typeField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "type", value.name()); + _typeField = value; + } + break; + } + return this; + } + + /** + * Setter for type + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Dimension.Fields#type + */ + public Dimension setType( + @Nonnull + DimensionType value) { + if (value == null) { + throw new NullPointerException("Cannot set field type of com.linkedin.feathr.compute.Dimension to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "type", value.name()); + _typeField = value; + } + return this; + } + + /** + * Existence checker for shape + * + * @see Dimension.Fields#shape + */ + public boolean hasShape() { + if (_shapeField!= null) { + return true; + } + return super._map.containsKey("shape"); + } + + /** + * Remover for shape + * + * @see Dimension.Fields#shape + */ + public void removeShape() { + super._map.remove("shape"); + } + + /** + * Getter for shape + * + * @see Dimension.Fields#shape + */ + public Integer getShape(GetMode mode) { + return getShape(); + } + + /** + * Getter for shape + * + * @return + * Optional field. Always check for null. + * @see Dimension.Fields#shape + */ + @Nullable + public Integer getShape() { + if (_shapeField!= null) { + return _shapeField; + } else { + Object __rawValue = super._map.get("shape"); + _shapeField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _shapeField; + } + } + + /** + * Setter for shape + * + * @see Dimension.Fields#shape + */ + public Dimension setShape(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setShape(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeShape(); + } else { + CheckedUtil.putWithoutChecking(super._map, "shape", DataTemplateUtil.coerceIntInput(value)); + _shapeField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "shape", DataTemplateUtil.coerceIntInput(value)); + _shapeField = value; + } + break; + } + return this; + } + + /** + * Setter for shape + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Dimension.Fields#shape + */ + public Dimension setShape( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field shape of com.linkedin.feathr.compute.Dimension to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "shape", DataTemplateUtil.coerceIntInput(value)); + _shapeField = value; + } + return this; + } + + /** + * Setter for shape + * + * @see Dimension.Fields#shape + */ + public Dimension setShape(int value) { + CheckedUtil.putWithoutChecking(super._map, "shape", DataTemplateUtil.coerceIntInput(value)); + _shapeField = value; + return this; + } + + @Override + public Dimension clone() + throws CloneNotSupportedException + { + Dimension __clone = ((Dimension) super.clone()); + __clone.__changeListener = new Dimension.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public Dimension copy() + throws CloneNotSupportedException + { + Dimension __copy = ((Dimension) super.copy()); + __copy._shapeField = null; + __copy._typeField = null; + __copy.__changeListener = new Dimension.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final Dimension __objectRef; + + private ChangeListener(Dimension reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "shape": + __objectRef._shapeField = null; + break; + case "type": + __objectRef._typeField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Type of the dimension in the tensor. Each dimension can have a different type. + * + */ + public PathSpec type() { + return new PathSpec(getPathComponents(), "type"); + } + + /** + * Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime. + * + */ + public PathSpec shape() { + return new PathSpec(getPathComponents(), "shape"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(3); + } + + /** + * Type of the dimension in the tensor. Each dimension can have a different type. + * + */ + public Dimension.ProjectionMask withType() { + getDataMap().put("type", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime. + * + */ + public Dimension.ProjectionMask withShape() { + getDataMap().put("shape", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionArray.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionArray.java new file mode 100644 index 000000000..78c173375 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionArray.java @@ -0,0 +1,118 @@ + +package com.linkedin.feathr.compute; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.schema.ArrayDataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.TemplateOutputCastException; +import com.linkedin.data.template.WrappingArrayTemplate; + +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\TensorFeatureFormat.pdl.") +public class DimensionArray + extends WrappingArrayTemplate +{ + + private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[{namespace com.linkedin.feathr.compute/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}}]", SchemaFormatType.PDL)); + + public DimensionArray() { + this(new DataList()); + } + + public DimensionArray(int initialCapacity) { + this(new DataList(initialCapacity)); + } + + public DimensionArray(Collection c) { + this(new DataList(c.size())); + addAll(c); + } + + public DimensionArray(DataList data) { + super(data, SCHEMA, Dimension.class); + } + + public DimensionArray(Dimension first, Dimension... rest) { + this(new DataList((rest.length + 1))); + add(first); + addAll(Arrays.asList(rest)); + } + + public static ArrayDataSchema dataSchema() { + return SCHEMA; + } + + public static DimensionArray.ProjectionMask createMask() { + return new DimensionArray.ProjectionMask(); + } + + @Override + public DimensionArray clone() + throws CloneNotSupportedException + { + DimensionArray __clone = ((DimensionArray) super.clone()); + return __clone; + } + + @Override + public DimensionArray copy() + throws CloneNotSupportedException + { + DimensionArray __copy = ((DimensionArray) super.copy()); + return __copy; + } + + @Override + protected Dimension coerceOutput(Object object) + throws TemplateOutputCastException + { + assert(object != null); + return ((object == null)?null:new Dimension(DataTemplateUtil.castOrThrow(object, DataMap.class))); + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.compute.Dimension.Fields items() { + return new com.linkedin.feathr.compute.Dimension.Fields(getPathComponents(), PathSpec.WILDCARD); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.Dimension.ProjectionMask _itemsMask; + + ProjectionMask() { + super(4); + } + + public DimensionArray.ProjectionMask withItems(Function nestedMask) { + _itemsMask = nestedMask.apply(((_itemsMask == null)?Dimension.createMask():_itemsMask)); + getDataMap().put("$*", _itemsMask.getDataMap()); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionType.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionType.java new file mode 100644 index 000000000..d996b4dcc --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionType.java @@ -0,0 +1,48 @@ + +package com.linkedin.feathr.compute; + +import javax.annotation.Generated; +import com.linkedin.data.schema.EnumDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; + + +/** + * Supported dimension types for tensors in Quince and feathr. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\DimensionType.pdl.") +public enum DimensionType { + + + /** + * Long. + * + */ + LONG, + + /** + * Integer. + * + */ + INT, + + /** + * String. + * + */ + STRING, + + /** + * Boolean. + * + */ + BOOLEAN, + $UNKNOWN; + private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}", SchemaFormatType.PDL)); + + public static EnumDataSchema dataSchema() { + return SCHEMA; + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/External.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/External.java new file mode 100644 index 000000000..1d956ea27 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/External.java @@ -0,0 +1,557 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * A temporary node which would exist only while parsing the graph. For example, when parsing an object if there is a reference to a feature + * name, we will create an external node. This would get resolved later in the computation. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\External.pdl.") +public class External + extends RecordTemplate +{ + + private final static External.Fields _fields = new External.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**A temporary node which would exist only while parsing the graph. For example, when parsing an object if there is a reference to a feature\r\nname, we will create an external node. This would get resolved later in the computation.*/record External includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**Name of the external object it should refer to.*/name:string}", SchemaFormatType.PDL)); + private Integer _idField = null; + private ConcreteKey _concreteKeyField = null; + private String _nameField = null; + private External.ChangeListener __changeListener = new External.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); + private final static RecordDataSchema.Field FIELD_ConcreteKey = SCHEMA.getField("concreteKey"); + private final static RecordDataSchema.Field FIELD_Name = SCHEMA.getField("name"); + + public External() { + super(new DataMap(4, 0.75F), SCHEMA, 2); + addChangeListener(__changeListener); + } + + public External(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static External.Fields fields() { + return _fields; + } + + public static External.ProjectionMask createMask() { + return new External.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for id + * + * @see External.Fields#id + */ + public boolean hasId() { + if (_idField!= null) { + return true; + } + return super._map.containsKey("id"); + } + + /** + * Remover for id + * + * @see External.Fields#id + */ + public void removeId() { + super._map.remove("id"); + } + + /** + * Getter for id + * + * @see External.Fields#id + */ + public Integer getId(GetMode mode) { + switch (mode) { + case STRICT: + return getId(); + case DEFAULT: + case NULL: + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for id + * + * @return + * Required field. Could be null for partial record. + * @see External.Fields#id + */ + @Nonnull + public Integer getId() { + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("id"); + } + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + + /** + * Setter for id + * + * @see External.Fields#id + */ + public External setId(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setId(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.External"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeId(); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + } + return this; + } + + /** + * Setter for id + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see External.Fields#id + */ + public External setId( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.External to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + return this; + } + + /** + * Setter for id + * + * @see External.Fields#id + */ + public External setId(int value) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + return this; + } + + /** + * Existence checker for concreteKey + * + * @see External.Fields#concreteKey + */ + public boolean hasConcreteKey() { + if (_concreteKeyField!= null) { + return true; + } + return super._map.containsKey("concreteKey"); + } + + /** + * Remover for concreteKey + * + * @see External.Fields#concreteKey + */ + public void removeConcreteKey() { + super._map.remove("concreteKey"); + } + + /** + * Getter for concreteKey + * + * @see External.Fields#concreteKey + */ + public ConcreteKey getConcreteKey(GetMode mode) { + return getConcreteKey(); + } + + /** + * Getter for concreteKey + * + * @return + * Optional field. Always check for null. + * @see External.Fields#concreteKey + */ + @Nullable + public ConcreteKey getConcreteKey() { + if (_concreteKeyField!= null) { + return _concreteKeyField; + } else { + Object __rawValue = super._map.get("concreteKey"); + _concreteKeyField = ((__rawValue == null)?null:new ConcreteKey(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _concreteKeyField; + } + } + + /** + * Setter for concreteKey + * + * @see External.Fields#concreteKey + */ + public External setConcreteKey(ConcreteKey value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setConcreteKey(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeConcreteKey(); + } else { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + break; + } + return this; + } + + /** + * Setter for concreteKey + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see External.Fields#concreteKey + */ + public External setConcreteKey( + @Nonnull + ConcreteKey value) { + if (value == null) { + throw new NullPointerException("Cannot set field concreteKey of com.linkedin.feathr.compute.External to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + return this; + } + + /** + * Existence checker for name + * + * @see External.Fields#name + */ + public boolean hasName() { + if (_nameField!= null) { + return true; + } + return super._map.containsKey("name"); + } + + /** + * Remover for name + * + * @see External.Fields#name + */ + public void removeName() { + super._map.remove("name"); + } + + /** + * Getter for name + * + * @see External.Fields#name + */ + public String getName(GetMode mode) { + switch (mode) { + case STRICT: + return getName(); + case DEFAULT: + case NULL: + if (_nameField!= null) { + return _nameField; + } else { + Object __rawValue = super._map.get("name"); + _nameField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _nameField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for name + * + * @return + * Required field. Could be null for partial record. + * @see External.Fields#name + */ + @Nonnull + public String getName() { + if (_nameField!= null) { + return _nameField; + } else { + Object __rawValue = super._map.get("name"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("name"); + } + _nameField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _nameField; + } + } + + /** + * Setter for name + * + * @see External.Fields#name + */ + public External setName(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setName(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field name of com.linkedin.feathr.compute.External"); + } else { + CheckedUtil.putWithoutChecking(super._map, "name", value); + _nameField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeName(); + } else { + CheckedUtil.putWithoutChecking(super._map, "name", value); + _nameField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "name", value); + _nameField = value; + } + break; + } + return this; + } + + /** + * Setter for name + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see External.Fields#name + */ + public External setName( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field name of com.linkedin.feathr.compute.External to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "name", value); + _nameField = value; + } + return this; + } + + @Override + public External clone() + throws CloneNotSupportedException + { + External __clone = ((External) super.clone()); + __clone.__changeListener = new External.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public External copy() + throws CloneNotSupportedException + { + External __copy = ((External) super.copy()); + __copy._nameField = null; + __copy._idField = null; + __copy._concreteKeyField = null; + __copy.__changeListener = new External.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final External __objectRef; + + private ChangeListener(External reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "name": + __objectRef._nameField = null; + break; + case "id": + __objectRef._idField = null; + break; + case "concreteKey": + __objectRef._concreteKeyField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The node would be represented by this id. + * + */ + public PathSpec id() { + return new PathSpec(getPathComponents(), "id"); + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public com.linkedin.feathr.compute.ConcreteKey.Fields concreteKey() { + return new com.linkedin.feathr.compute.ConcreteKey.Fields(getPathComponents(), "concreteKey"); + } + + /** + * Name of the external object it should refer to. + * + */ + public PathSpec name() { + return new PathSpec(getPathComponents(), "name"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.ConcreteKey.ProjectionMask _concreteKeyMask; + + ProjectionMask() { + super(4); + } + + /** + * The node would be represented by this id. + * + */ + public External.ProjectionMask withId() { + getDataMap().put("id", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public External.ProjectionMask withConcreteKey(Function nestedMask) { + _concreteKeyMask = nestedMask.apply(((_concreteKeyMask == null)?ConcreteKey.createMask():_concreteKeyMask)); + getDataMap().put("concreteKey", _concreteKeyMask.getDataMap()); + return this; + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public External.ProjectionMask withConcreteKey() { + _concreteKeyMask = null; + getDataMap().put("concreteKey", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Name of the external object it should refer to. + * + */ + public External.ProjectionMask withName() { + getDataMap().put("name", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureValue.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureValue.java new file mode 100644 index 000000000..24f400003 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureValue.java @@ -0,0 +1,431 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import javax.annotation.Generated; +import com.linkedin.data.ByteString; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.DataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.TyperefDataSchema; +import com.linkedin.data.schema.UnionDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.HasTyperefInfo; +import com.linkedin.data.template.TyperefInfo; +import com.linkedin.data.template.UnionTemplate; + +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\FeatureValue.pdl.") +public class FeatureValue + extends UnionTemplate + implements HasTyperefInfo +{ + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[boolean,int,long,float,double,string,bytes]", SchemaFormatType.PDL)); + private java.lang.Boolean _booleanMember = null; + private Integer _intMember = null; + private java.lang.Long _longMember = null; + private java.lang.Float _floatMember = null; + private java.lang.Double _doubleMember = null; + private java.lang.String _stringMember = null; + private ByteString _bytesMember = null; + private FeatureValue.ChangeListener __changeListener = new FeatureValue.ChangeListener(this); + private final static DataSchema MEMBER_Boolean = SCHEMA.getTypeByMemberKey("boolean"); + private final static DataSchema MEMBER_Int = SCHEMA.getTypeByMemberKey("int"); + private final static DataSchema MEMBER_Long = SCHEMA.getTypeByMemberKey("long"); + private final static DataSchema MEMBER_Float = SCHEMA.getTypeByMemberKey("float"); + private final static DataSchema MEMBER_Double = SCHEMA.getTypeByMemberKey("double"); + private final static DataSchema MEMBER_String = SCHEMA.getTypeByMemberKey("string"); + private final static DataSchema MEMBER_Bytes = SCHEMA.getTypeByMemberKey("bytes"); + private final static TyperefInfo TYPEREFINFO = new FeatureValue.UnionTyperefInfo(); + + public FeatureValue() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public FeatureValue(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static FeatureValue create(java.lang.Boolean value) { + FeatureValue newUnion = new FeatureValue(); + newUnion.setBoolean(value); + return newUnion; + } + + public boolean isBoolean() { + return memberIs("boolean"); + } + + public java.lang.Boolean getBoolean() { + checkNotNull(); + if (_booleanMember!= null) { + return _booleanMember; + } + Object __rawValue = super._map.get("boolean"); + _booleanMember = DataTemplateUtil.coerceBooleanOutput(__rawValue); + return _booleanMember; + } + + public void setBoolean(java.lang.Boolean value) { + checkNotNull(); + super._map.clear(); + _booleanMember = value; + CheckedUtil.putWithoutChecking(super._map, "boolean", value); + } + + public static FeatureValue create(Integer value) { + FeatureValue newUnion = new FeatureValue(); + newUnion.setInt(value); + return newUnion; + } + + public boolean isInt() { + return memberIs("int"); + } + + public Integer getInt() { + checkNotNull(); + if (_intMember!= null) { + return _intMember; + } + Object __rawValue = super._map.get("int"); + _intMember = DataTemplateUtil.coerceIntOutput(__rawValue); + return _intMember; + } + + public void setInt(Integer value) { + checkNotNull(); + super._map.clear(); + _intMember = value; + CheckedUtil.putWithoutChecking(super._map, "int", DataTemplateUtil.coerceIntInput(value)); + } + + public static FeatureValue create(java.lang.Long value) { + FeatureValue newUnion = new FeatureValue(); + newUnion.setLong(value); + return newUnion; + } + + public boolean isLong() { + return memberIs("long"); + } + + public java.lang.Long getLong() { + checkNotNull(); + if (_longMember!= null) { + return _longMember; + } + Object __rawValue = super._map.get("long"); + _longMember = DataTemplateUtil.coerceLongOutput(__rawValue); + return _longMember; + } + + public void setLong(java.lang.Long value) { + checkNotNull(); + super._map.clear(); + _longMember = value; + CheckedUtil.putWithoutChecking(super._map, "long", DataTemplateUtil.coerceLongInput(value)); + } + + public static FeatureValue create(java.lang.Float value) { + FeatureValue newUnion = new FeatureValue(); + newUnion.setFloat(value); + return newUnion; + } + + public boolean isFloat() { + return memberIs("float"); + } + + public java.lang.Float getFloat() { + checkNotNull(); + if (_floatMember!= null) { + return _floatMember; + } + Object __rawValue = super._map.get("float"); + _floatMember = DataTemplateUtil.coerceFloatOutput(__rawValue); + return _floatMember; + } + + public void setFloat(java.lang.Float value) { + checkNotNull(); + super._map.clear(); + _floatMember = value; + CheckedUtil.putWithoutChecking(super._map, "float", DataTemplateUtil.coerceFloatInput(value)); + } + + public static FeatureValue create(java.lang.Double value) { + FeatureValue newUnion = new FeatureValue(); + newUnion.setDouble(value); + return newUnion; + } + + public boolean isDouble() { + return memberIs("double"); + } + + public java.lang.Double getDouble() { + checkNotNull(); + if (_doubleMember!= null) { + return _doubleMember; + } + Object __rawValue = super._map.get("double"); + _doubleMember = DataTemplateUtil.coerceDoubleOutput(__rawValue); + return _doubleMember; + } + + public void setDouble(java.lang.Double value) { + checkNotNull(); + super._map.clear(); + _doubleMember = value; + CheckedUtil.putWithoutChecking(super._map, "double", DataTemplateUtil.coerceDoubleInput(value)); + } + + public static FeatureValue create(java.lang.String value) { + FeatureValue newUnion = new FeatureValue(); + newUnion.setString(value); + return newUnion; + } + + public boolean isString() { + return memberIs("string"); + } + + public java.lang.String getString() { + checkNotNull(); + if (_stringMember!= null) { + return _stringMember; + } + Object __rawValue = super._map.get("string"); + _stringMember = DataTemplateUtil.coerceStringOutput(__rawValue); + return _stringMember; + } + + public void setString(java.lang.String value) { + checkNotNull(); + super._map.clear(); + _stringMember = value; + CheckedUtil.putWithoutChecking(super._map, "string", value); + } + + public static FeatureValue create(ByteString value) { + FeatureValue newUnion = new FeatureValue(); + newUnion.setBytes(value); + return newUnion; + } + + public boolean isBytes() { + return memberIs("bytes"); + } + + public ByteString getBytes() { + checkNotNull(); + if (_bytesMember!= null) { + return _bytesMember; + } + Object __rawValue = super._map.get("bytes"); + _bytesMember = DataTemplateUtil.coerceBytesOutput(__rawValue); + return _bytesMember; + } + + public void setBytes(ByteString value) { + checkNotNull(); + super._map.clear(); + _bytesMember = value; + CheckedUtil.putWithoutChecking(super._map, "bytes", value); + } + + public static FeatureValue.ProjectionMask createMask() { + return new FeatureValue.ProjectionMask(); + } + + @Override + public FeatureValue clone() + throws CloneNotSupportedException + { + FeatureValue __clone = ((FeatureValue) super.clone()); + __clone.__changeListener = new FeatureValue.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public FeatureValue copy() + throws CloneNotSupportedException + { + FeatureValue __copy = ((FeatureValue) super.copy()); + __copy._booleanMember = null; + __copy._stringMember = null; + __copy._doubleMember = null; + __copy._bytesMember = null; + __copy._floatMember = null; + __copy._intMember = null; + __copy._longMember = null; + __copy.__changeListener = new FeatureValue.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + public TyperefInfo typerefInfo() { + return TYPEREFINFO; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final FeatureValue __objectRef; + + private ChangeListener(FeatureValue reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(java.lang.String key, Object value) { + switch (key) { + case "boolean": + __objectRef._booleanMember = null; + break; + case "string": + __objectRef._stringMember = null; + break; + case "double": + __objectRef._doubleMember = null; + break; + case "bytes": + __objectRef._bytesMember = null; + break; + case "float": + __objectRef._floatMember = null; + break; + case "int": + __objectRef._intMember = null; + break; + case "long": + __objectRef._longMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, java.lang.String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public PathSpec Boolean() { + return new PathSpec(getPathComponents(), "boolean"); + } + + public PathSpec Int() { + return new PathSpec(getPathComponents(), "int"); + } + + public PathSpec Long() { + return new PathSpec(getPathComponents(), "long"); + } + + public PathSpec Float() { + return new PathSpec(getPathComponents(), "float"); + } + + public PathSpec Double() { + return new PathSpec(getPathComponents(), "double"); + } + + public PathSpec String() { + return new PathSpec(getPathComponents(), "string"); + } + + public PathSpec Bytes() { + return new PathSpec(getPathComponents(), "bytes"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(10); + } + + public FeatureValue.ProjectionMask withBoolean() { + getDataMap().put("boolean", MaskMap.POSITIVE_MASK); + return this; + } + + public FeatureValue.ProjectionMask withInt() { + getDataMap().put("int", MaskMap.POSITIVE_MASK); + return this; + } + + public FeatureValue.ProjectionMask withLong() { + getDataMap().put("long", MaskMap.POSITIVE_MASK); + return this; + } + + public FeatureValue.ProjectionMask withFloat() { + getDataMap().put("float", MaskMap.POSITIVE_MASK); + return this; + } + + public FeatureValue.ProjectionMask withDouble() { + getDataMap().put("double", MaskMap.POSITIVE_MASK); + return this; + } + + public FeatureValue.ProjectionMask withString() { + getDataMap().put("string", MaskMap.POSITIVE_MASK); + return this; + } + + public FeatureValue.ProjectionMask withBytes() { + getDataMap().put("bytes", MaskMap.POSITIVE_MASK); + return this; + } + + } + + + /** + * Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases. + * + */ + private final static class UnionTyperefInfo + extends TyperefInfo + { + + private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]", SchemaFormatType.PDL)); + + public UnionTyperefInfo() { + super(SCHEMA); + } + + public static TyperefDataSchema dataSchema() { + return SCHEMA; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureVersion.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureVersion.java new file mode 100644 index 000000000..432fa6410 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureVersion.java @@ -0,0 +1,526 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.SetMode; + + +/** + * + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\FeatureVersion.pdl.") +public class FeatureVersion + extends RecordTemplate +{ + + private final static FeatureVersion.Fields _fields = new FeatureVersion.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute,record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}", SchemaFormatType.PDL)); + private FrameFeatureType _typeField = null; + private TensorFeatureFormat _formatField = null; + private FeatureValue _defaultValueField = null; + private FeatureVersion.ChangeListener __changeListener = new FeatureVersion.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Type = SCHEMA.getField("type"); + private final static FrameFeatureType DEFAULT_Type; + private final static RecordDataSchema.Field FIELD_Format = SCHEMA.getField("format"); + private final static RecordDataSchema.Field FIELD_DefaultValue = SCHEMA.getField("defaultValue"); + + static { + DEFAULT_Type = DataTemplateUtil.coerceEnumOutput(FIELD_Type.getDefault(), FrameFeatureType.class, FrameFeatureType.$UNKNOWN); + } + + public FeatureVersion() { + super(new DataMap(4, 0.75F), SCHEMA, 3); + addChangeListener(__changeListener); + } + + public FeatureVersion(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static FeatureVersion.Fields fields() { + return _fields; + } + + public static FeatureVersion.ProjectionMask createMask() { + return new FeatureVersion.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for type + * + * @see FeatureVersion.Fields#type + */ + public boolean hasType() { + if (_typeField!= null) { + return true; + } + return super._map.containsKey("type"); + } + + /** + * Remover for type + * + * @see FeatureVersion.Fields#type + */ + public void removeType() { + super._map.remove("type"); + } + + /** + * Getter for type + * + * @see FeatureVersion.Fields#type + */ + public FrameFeatureType getType(GetMode mode) { + switch (mode) { + case STRICT: + case DEFAULT: + return getType(); + case NULL: + if (_typeField!= null) { + return _typeField; + } else { + Object __rawValue = super._map.get("type"); + _typeField = DataTemplateUtil.coerceEnumOutput(__rawValue, FrameFeatureType.class, FrameFeatureType.$UNKNOWN); + return _typeField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for type + * + * @return + * Required field. Could be null for partial record. + * @see FeatureVersion.Fields#type + */ + @Nonnull + public FrameFeatureType getType() { + if (_typeField!= null) { + return _typeField; + } else { + Object __rawValue = super._map.get("type"); + if (__rawValue == null) { + return DEFAULT_Type; + } + _typeField = DataTemplateUtil.coerceEnumOutput(__rawValue, FrameFeatureType.class, FrameFeatureType.$UNKNOWN); + return _typeField; + } + } + + /** + * Setter for type + * + * @see FeatureVersion.Fields#type + */ + public FeatureVersion setType(FrameFeatureType value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setType(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field type of com.linkedin.feathr.compute.FeatureVersion"); + } else { + CheckedUtil.putWithoutChecking(super._map, "type", value.name()); + _typeField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeType(); + } else { + CheckedUtil.putWithoutChecking(super._map, "type", value.name()); + _typeField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "type", value.name()); + _typeField = value; + } + break; + } + return this; + } + + /** + * Setter for type + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see FeatureVersion.Fields#type + */ + public FeatureVersion setType( + @Nonnull + FrameFeatureType value) { + if (value == null) { + throw new NullPointerException("Cannot set field type of com.linkedin.feathr.compute.FeatureVersion to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "type", value.name()); + _typeField = value; + } + return this; + } + + /** + * Existence checker for format + * + * @see FeatureVersion.Fields#format + */ + public boolean hasFormat() { + if (_formatField!= null) { + return true; + } + return super._map.containsKey("format"); + } + + /** + * Remover for format + * + * @see FeatureVersion.Fields#format + */ + public void removeFormat() { + super._map.remove("format"); + } + + /** + * Getter for format + * + * @see FeatureVersion.Fields#format + */ + public TensorFeatureFormat getFormat(GetMode mode) { + return getFormat(); + } + + /** + * Getter for format + * + * @return + * Optional field. Always check for null. + * @see FeatureVersion.Fields#format + */ + @Nullable + public TensorFeatureFormat getFormat() { + if (_formatField!= null) { + return _formatField; + } else { + Object __rawValue = super._map.get("format"); + _formatField = ((__rawValue == null)?null:new TensorFeatureFormat(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _formatField; + } + } + + /** + * Setter for format + * + * @see FeatureVersion.Fields#format + */ + public FeatureVersion setFormat(TensorFeatureFormat value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFormat(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeFormat(); + } else { + CheckedUtil.putWithoutChecking(super._map, "format", value.data()); + _formatField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "format", value.data()); + _formatField = value; + } + break; + } + return this; + } + + /** + * Setter for format + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see FeatureVersion.Fields#format + */ + public FeatureVersion setFormat( + @Nonnull + TensorFeatureFormat value) { + if (value == null) { + throw new NullPointerException("Cannot set field format of com.linkedin.feathr.compute.FeatureVersion to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "format", value.data()); + _formatField = value; + } + return this; + } + + /** + * Existence checker for defaultValue + * + * @see FeatureVersion.Fields#defaultValue + */ + public boolean hasDefaultValue() { + if (_defaultValueField!= null) { + return true; + } + return super._map.containsKey("defaultValue"); + } + + /** + * Remover for defaultValue + * + * @see FeatureVersion.Fields#defaultValue + */ + public void removeDefaultValue() { + super._map.remove("defaultValue"); + } + + /** + * Getter for defaultValue + * + * @see FeatureVersion.Fields#defaultValue + */ + public FeatureValue getDefaultValue(GetMode mode) { + return getDefaultValue(); + } + + /** + * Getter for defaultValue + * + * @return + * Optional field. Always check for null. + * @see FeatureVersion.Fields#defaultValue + */ + @Nullable + public FeatureValue getDefaultValue() { + if (_defaultValueField!= null) { + return _defaultValueField; + } else { + Object __rawValue = super._map.get("defaultValue"); + _defaultValueField = ((__rawValue == null)?null:new FeatureValue(__rawValue)); + return _defaultValueField; + } + } + + /** + * Setter for defaultValue + * + * @see FeatureVersion.Fields#defaultValue + */ + public FeatureVersion setDefaultValue(FeatureValue value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setDefaultValue(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeDefaultValue(); + } else { + CheckedUtil.putWithoutChecking(super._map, "defaultValue", value.data()); + _defaultValueField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "defaultValue", value.data()); + _defaultValueField = value; + } + break; + } + return this; + } + + /** + * Setter for defaultValue + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see FeatureVersion.Fields#defaultValue + */ + public FeatureVersion setDefaultValue( + @Nonnull + FeatureValue value) { + if (value == null) { + throw new NullPointerException("Cannot set field defaultValue of com.linkedin.feathr.compute.FeatureVersion to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "defaultValue", value.data()); + _defaultValueField = value; + } + return this; + } + + @Override + public FeatureVersion clone() + throws CloneNotSupportedException + { + FeatureVersion __clone = ((FeatureVersion) super.clone()); + __clone.__changeListener = new FeatureVersion.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public FeatureVersion copy() + throws CloneNotSupportedException + { + FeatureVersion __copy = ((FeatureVersion) super.copy()); + __copy._defaultValueField = null; + __copy._formatField = null; + __copy._typeField = null; + __copy.__changeListener = new FeatureVersion.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final FeatureVersion __objectRef; + + private ChangeListener(FeatureVersion reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "defaultValue": + __objectRef._defaultValueField = null; + break; + case "format": + __objectRef._formatField = null; + break; + case "type": + __objectRef._typeField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed + * + */ + public PathSpec type() { + return new PathSpec(getPathComponents(), "type"); + } + + /** + * Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features. + * + */ + public com.linkedin.feathr.compute.TensorFeatureFormat.Fields format() { + return new com.linkedin.feathr.compute.TensorFeatureFormat.Fields(getPathComponents(), "format"); + } + + /** + * An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data. + * + */ + public com.linkedin.feathr.compute.FeatureValue.Fields defaultValue() { + return new com.linkedin.feathr.compute.FeatureValue.Fields(getPathComponents(), "defaultValue"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.TensorFeatureFormat.ProjectionMask _formatMask; + private com.linkedin.feathr.compute.FeatureValue.ProjectionMask _defaultValueMask; + + ProjectionMask() { + super(4); + } + + /** + * Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed + * + */ + public FeatureVersion.ProjectionMask withType() { + getDataMap().put("type", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features. + * + */ + public FeatureVersion.ProjectionMask withFormat(Function nestedMask) { + _formatMask = nestedMask.apply(((_formatMask == null)?TensorFeatureFormat.createMask():_formatMask)); + getDataMap().put("format", _formatMask.getDataMap()); + return this; + } + + /** + * Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features. + * + */ + public FeatureVersion.ProjectionMask withFormat() { + _formatMask = null; + getDataMap().put("format", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data. + * + */ + public FeatureVersion.ProjectionMask withDefaultValue(Function nestedMask) { + _defaultValueMask = nestedMask.apply(((_defaultValueMask == null)?FeatureValue.createMask():_defaultValueMask)); + getDataMap().put("defaultValue", _defaultValueMask.getDataMap()); + return this; + } + + /** + * An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data. + * + */ + public FeatureVersion.ProjectionMask withDefaultValue() { + _defaultValueMask = null; + getDataMap().put("defaultValue", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FrameFeatureType.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FrameFeatureType.java new file mode 100644 index 000000000..f92056e89 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FrameFeatureType.java @@ -0,0 +1,72 @@ + +package com.linkedin.feathr.compute; + +import javax.annotation.Generated; +import com.linkedin.data.schema.EnumDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; + + +/** + * The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\FrameFeatureType.pdl.") +public enum FrameFeatureType { + + + /** + * Boolean valued feature + * + */ + BOOLEAN, + + /** + * Numerically valued feature such as INT, LONG, DOUBLE, etc + * + */ + NUMERIC, + + /** + * Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) + * + */ + CATEGORICAL, + + /** + * Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) + * + */ + CATEGORICAL_SET, + + /** + * Represents a feature in vector format where the the majority of the elements are non-zero + * + */ + DENSE_VECTOR, + + /** + * Represents features that has string terms and numeric value + * + */ + TERM_VECTOR, + + /** + * Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML). + * + */ + TENSOR, + + /** + * Placeholder for when no types are specified + * + */ + UNSPECIFIED, + $UNKNOWN; + private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}", SchemaFormatType.PDL)); + + public static EnumDataSchema dataSchema() { + return SCHEMA; + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyExpressionType.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyExpressionType.java new file mode 100644 index 000000000..a8bbfd957 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyExpressionType.java @@ -0,0 +1,44 @@ + +package com.linkedin.feathr.compute; + +import javax.annotation.Generated; +import com.linkedin.data.schema.EnumDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; + + +/** + * Different key formats supported. + * Todo - We probably do not want to generalize this as a kind of key-operator in the core compute model, + * with instances such as for MVEL or SQL being available (e.g. via an OperatorId reference). + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\KeyExpressionType.pdl.") +public enum KeyExpressionType { + + + /** + * Java-based MVEL + * + */ + MVEL, + + /** + * Spark-SQL + * + */ + SQL, + + /** + * Custom java/scala UDF + * + */ + UDF, + $UNKNOWN; + private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Different key formats supported.\r\nTodo - We probably do not want to generalize this as a kind of key-operator in the core compute model,\r\nwith instances such as for MVEL or SQL being available (e.g. via an OperatorId reference).*/enum KeyExpressionType{/**Java-based MVEL*/MVEL/**Spark-SQL*/SQL/**Custom java/scala UDF*/UDF}", SchemaFormatType.PDL)); + + public static EnumDataSchema dataSchema() { + return SCHEMA; + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReference.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReference.java new file mode 100644 index 000000000..887c4aac8 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReference.java @@ -0,0 +1,272 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * This represents the position of the key in the node which is being referred to. For example, if the original node has a key + * like [x, y], and the keyReference says 1, it is referring to y. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\KeyReference.pdl.") +public class KeyReference + extends RecordTemplate +{ + + private final static KeyReference.Fields _fields = new KeyReference.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}", SchemaFormatType.PDL)); + private Integer _positionField = null; + private KeyReference.ChangeListener __changeListener = new KeyReference.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Position = SCHEMA.getField("position"); + + public KeyReference() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public KeyReference(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static KeyReference.Fields fields() { + return _fields; + } + + public static KeyReference.ProjectionMask createMask() { + return new KeyReference.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for position + * + * @see KeyReference.Fields#position + */ + public boolean hasPosition() { + if (_positionField!= null) { + return true; + } + return super._map.containsKey("position"); + } + + /** + * Remover for position + * + * @see KeyReference.Fields#position + */ + public void removePosition() { + super._map.remove("position"); + } + + /** + * Getter for position + * + * @see KeyReference.Fields#position + */ + public Integer getPosition(GetMode mode) { + switch (mode) { + case STRICT: + return getPosition(); + case DEFAULT: + case NULL: + if (_positionField!= null) { + return _positionField; + } else { + Object __rawValue = super._map.get("position"); + _positionField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _positionField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for position + * + * @return + * Required field. Could be null for partial record. + * @see KeyReference.Fields#position + */ + @Nonnull + public Integer getPosition() { + if (_positionField!= null) { + return _positionField; + } else { + Object __rawValue = super._map.get("position"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("position"); + } + _positionField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _positionField; + } + } + + /** + * Setter for position + * + * @see KeyReference.Fields#position + */ + public KeyReference setPosition(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setPosition(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field position of com.linkedin.feathr.compute.KeyReference"); + } else { + CheckedUtil.putWithoutChecking(super._map, "position", DataTemplateUtil.coerceIntInput(value)); + _positionField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removePosition(); + } else { + CheckedUtil.putWithoutChecking(super._map, "position", DataTemplateUtil.coerceIntInput(value)); + _positionField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "position", DataTemplateUtil.coerceIntInput(value)); + _positionField = value; + } + break; + } + return this; + } + + /** + * Setter for position + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see KeyReference.Fields#position + */ + public KeyReference setPosition( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field position of com.linkedin.feathr.compute.KeyReference to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "position", DataTemplateUtil.coerceIntInput(value)); + _positionField = value; + } + return this; + } + + /** + * Setter for position + * + * @see KeyReference.Fields#position + */ + public KeyReference setPosition(int value) { + CheckedUtil.putWithoutChecking(super._map, "position", DataTemplateUtil.coerceIntInput(value)); + _positionField = value; + return this; + } + + @Override + public KeyReference clone() + throws CloneNotSupportedException + { + KeyReference __clone = ((KeyReference) super.clone()); + __clone.__changeListener = new KeyReference.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public KeyReference copy() + throws CloneNotSupportedException + { + KeyReference __copy = ((KeyReference) super.copy()); + __copy._positionField = null; + __copy.__changeListener = new KeyReference.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final KeyReference __objectRef; + + private ChangeListener(KeyReference reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "position": + __objectRef._positionField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Position in the original key array + * + */ + public PathSpec position() { + return new PathSpec(getPathComponents(), "position"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(2); + } + + /** + * Position in the original key array + * + */ + public KeyReference.ProjectionMask withPosition() { + getDataMap().put("position", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReferenceArray.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReferenceArray.java new file mode 100644 index 000000000..df9bfe3c5 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReferenceArray.java @@ -0,0 +1,118 @@ + +package com.linkedin.feathr.compute; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.schema.ArrayDataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.TemplateOutputCastException; +import com.linkedin.data.template.WrappingArrayTemplate; + +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\NodeReference.pdl.") +public class KeyReferenceArray + extends WrappingArrayTemplate +{ + + private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[{namespace com.linkedin.feathr.compute/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}}]", SchemaFormatType.PDL)); + + public KeyReferenceArray() { + this(new DataList()); + } + + public KeyReferenceArray(int initialCapacity) { + this(new DataList(initialCapacity)); + } + + public KeyReferenceArray(Collection c) { + this(new DataList(c.size())); + addAll(c); + } + + public KeyReferenceArray(DataList data) { + super(data, SCHEMA, KeyReference.class); + } + + public KeyReferenceArray(KeyReference first, KeyReference... rest) { + this(new DataList((rest.length + 1))); + add(first); + addAll(Arrays.asList(rest)); + } + + public static ArrayDataSchema dataSchema() { + return SCHEMA; + } + + public static KeyReferenceArray.ProjectionMask createMask() { + return new KeyReferenceArray.ProjectionMask(); + } + + @Override + public KeyReferenceArray clone() + throws CloneNotSupportedException + { + KeyReferenceArray __clone = ((KeyReferenceArray) super.clone()); + return __clone; + } + + @Override + public KeyReferenceArray copy() + throws CloneNotSupportedException + { + KeyReferenceArray __copy = ((KeyReferenceArray) super.copy()); + return __copy; + } + + @Override + protected KeyReference coerceOutput(Object object) + throws TemplateOutputCastException + { + assert(object != null); + return ((object == null)?null:new KeyReference(DataTemplateUtil.castOrThrow(object, DataMap.class))); + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.compute.KeyReference.Fields items() { + return new com.linkedin.feathr.compute.KeyReference.Fields(getPathComponents(), PathSpec.WILDCARD); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.KeyReference.ProjectionMask _itemsMask; + + ProjectionMask() { + super(4); + } + + public KeyReferenceArray.ProjectionMask withItems(Function nestedMask) { + _itemsMask = nestedMask.apply(((_itemsMask == null)?KeyReference.createMask():_itemsMask)); + getDataMap().put("$*", _itemsMask.getDataMap()); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralView.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralView.java new file mode 100644 index 000000000..7cf602ffa --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralView.java @@ -0,0 +1,553 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.DataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.UnionDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.UnionTemplate; + + +/** + * Lateral view is used in conjunction with table generating functions (eg. the most commonly used explode()), which typically generates zero or more output rows for each input row. A lateral view first applies the table generating function to each row of base table, and then joins resulting output rows to the input rows to form a virtual table with the supplied table alias. For more details and examples, refer to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+LateralView. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\LateralView.pdl.") +public class LateralView + extends RecordTemplate +{ + + private final static LateralView.Fields _fields = new LateralView.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Lateral view is used in conjunction with table generating functions (eg. the most commonly used explode()), which typically generates zero or more output rows for each input row. A lateral view first applies the table generating function to each row of base table, and then joins resulting output rows to the input rows to form a virtual table with the supplied table alias. For more details and examples, refer to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+LateralView.*/record LateralView{/**A table-generating function transforms a single input row to multiple output rows. For example, explode(array('A','B','C') will produce 3 one-column rows, which are row1: 'A'; row2: 'B'; row3: 'C'.*/tableGeneratingFunction:union[/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}]/**Represents the alias for referencing the generated virtual table. It will be used in subsequent statements (eg. filter, groupBy) in the sliding window feature definition.*/virtualTableAlias:string}", SchemaFormatType.PDL)); + private LateralView.TableGeneratingFunction _tableGeneratingFunctionField = null; + private String _virtualTableAliasField = null; + private LateralView.ChangeListener __changeListener = new LateralView.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_TableGeneratingFunction = SCHEMA.getField("tableGeneratingFunction"); + private final static RecordDataSchema.Field FIELD_VirtualTableAlias = SCHEMA.getField("virtualTableAlias"); + + public LateralView() { + super(new DataMap(3, 0.75F), SCHEMA, 2); + addChangeListener(__changeListener); + } + + public LateralView(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static LateralView.Fields fields() { + return _fields; + } + + public static LateralView.ProjectionMask createMask() { + return new LateralView.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for tableGeneratingFunction + * + * @see LateralView.Fields#tableGeneratingFunction + */ + public boolean hasTableGeneratingFunction() { + if (_tableGeneratingFunctionField!= null) { + return true; + } + return super._map.containsKey("tableGeneratingFunction"); + } + + /** + * Remover for tableGeneratingFunction + * + * @see LateralView.Fields#tableGeneratingFunction + */ + public void removeTableGeneratingFunction() { + super._map.remove("tableGeneratingFunction"); + } + + /** + * Getter for tableGeneratingFunction + * + * @see LateralView.Fields#tableGeneratingFunction + */ + public LateralView.TableGeneratingFunction getTableGeneratingFunction(GetMode mode) { + switch (mode) { + case STRICT: + return getTableGeneratingFunction(); + case DEFAULT: + case NULL: + if (_tableGeneratingFunctionField!= null) { + return _tableGeneratingFunctionField; + } else { + Object __rawValue = super._map.get("tableGeneratingFunction"); + _tableGeneratingFunctionField = ((__rawValue == null)?null:new LateralView.TableGeneratingFunction(__rawValue)); + return _tableGeneratingFunctionField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for tableGeneratingFunction + * + * @return + * Required field. Could be null for partial record. + * @see LateralView.Fields#tableGeneratingFunction + */ + @Nonnull + public LateralView.TableGeneratingFunction getTableGeneratingFunction() { + if (_tableGeneratingFunctionField!= null) { + return _tableGeneratingFunctionField; + } else { + Object __rawValue = super._map.get("tableGeneratingFunction"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("tableGeneratingFunction"); + } + _tableGeneratingFunctionField = ((__rawValue == null)?null:new LateralView.TableGeneratingFunction(__rawValue)); + return _tableGeneratingFunctionField; + } + } + + /** + * Setter for tableGeneratingFunction + * + * @see LateralView.Fields#tableGeneratingFunction + */ + public LateralView setTableGeneratingFunction(LateralView.TableGeneratingFunction value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setTableGeneratingFunction(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field tableGeneratingFunction of com.linkedin.feathr.compute.LateralView"); + } else { + CheckedUtil.putWithoutChecking(super._map, "tableGeneratingFunction", value.data()); + _tableGeneratingFunctionField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeTableGeneratingFunction(); + } else { + CheckedUtil.putWithoutChecking(super._map, "tableGeneratingFunction", value.data()); + _tableGeneratingFunctionField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "tableGeneratingFunction", value.data()); + _tableGeneratingFunctionField = value; + } + break; + } + return this; + } + + /** + * Setter for tableGeneratingFunction + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see LateralView.Fields#tableGeneratingFunction + */ + public LateralView setTableGeneratingFunction( + @Nonnull + LateralView.TableGeneratingFunction value) { + if (value == null) { + throw new NullPointerException("Cannot set field tableGeneratingFunction of com.linkedin.feathr.compute.LateralView to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "tableGeneratingFunction", value.data()); + _tableGeneratingFunctionField = value; + } + return this; + } + + /** + * Existence checker for virtualTableAlias + * + * @see LateralView.Fields#virtualTableAlias + */ + public boolean hasVirtualTableAlias() { + if (_virtualTableAliasField!= null) { + return true; + } + return super._map.containsKey("virtualTableAlias"); + } + + /** + * Remover for virtualTableAlias + * + * @see LateralView.Fields#virtualTableAlias + */ + public void removeVirtualTableAlias() { + super._map.remove("virtualTableAlias"); + } + + /** + * Getter for virtualTableAlias + * + * @see LateralView.Fields#virtualTableAlias + */ + public String getVirtualTableAlias(GetMode mode) { + switch (mode) { + case STRICT: + return getVirtualTableAlias(); + case DEFAULT: + case NULL: + if (_virtualTableAliasField!= null) { + return _virtualTableAliasField; + } else { + Object __rawValue = super._map.get("virtualTableAlias"); + _virtualTableAliasField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _virtualTableAliasField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for virtualTableAlias + * + * @return + * Required field. Could be null for partial record. + * @see LateralView.Fields#virtualTableAlias + */ + @Nonnull + public String getVirtualTableAlias() { + if (_virtualTableAliasField!= null) { + return _virtualTableAliasField; + } else { + Object __rawValue = super._map.get("virtualTableAlias"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("virtualTableAlias"); + } + _virtualTableAliasField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _virtualTableAliasField; + } + } + + /** + * Setter for virtualTableAlias + * + * @see LateralView.Fields#virtualTableAlias + */ + public LateralView setVirtualTableAlias(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setVirtualTableAlias(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field virtualTableAlias of com.linkedin.feathr.compute.LateralView"); + } else { + CheckedUtil.putWithoutChecking(super._map, "virtualTableAlias", value); + _virtualTableAliasField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeVirtualTableAlias(); + } else { + CheckedUtil.putWithoutChecking(super._map, "virtualTableAlias", value); + _virtualTableAliasField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "virtualTableAlias", value); + _virtualTableAliasField = value; + } + break; + } + return this; + } + + /** + * Setter for virtualTableAlias + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see LateralView.Fields#virtualTableAlias + */ + public LateralView setVirtualTableAlias( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field virtualTableAlias of com.linkedin.feathr.compute.LateralView to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "virtualTableAlias", value); + _virtualTableAliasField = value; + } + return this; + } + + @Override + public LateralView clone() + throws CloneNotSupportedException + { + LateralView __clone = ((LateralView) super.clone()); + __clone.__changeListener = new LateralView.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public LateralView copy() + throws CloneNotSupportedException + { + LateralView __copy = ((LateralView) super.copy()); + __copy._virtualTableAliasField = null; + __copy._tableGeneratingFunctionField = null; + __copy.__changeListener = new LateralView.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final LateralView __objectRef; + + private ChangeListener(LateralView reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "virtualTableAlias": + __objectRef._virtualTableAliasField = null; + break; + case "tableGeneratingFunction": + __objectRef._tableGeneratingFunctionField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * A table-generating function transforms a single input row to multiple output rows. For example, explode(array('A','B','C') will produce 3 one-column rows, which are row1: 'A'; row2: 'B'; row3: 'C'. + * + */ + public com.linkedin.feathr.compute.LateralView.TableGeneratingFunction.Fields tableGeneratingFunction() { + return new com.linkedin.feathr.compute.LateralView.TableGeneratingFunction.Fields(getPathComponents(), "tableGeneratingFunction"); + } + + /** + * Represents the alias for referencing the generated virtual table. It will be used in subsequent statements (eg. filter, groupBy) in the sliding window feature definition. + * + */ + public PathSpec virtualTableAlias() { + return new PathSpec(getPathComponents(), "virtualTableAlias"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.LateralView.TableGeneratingFunction.ProjectionMask _tableGeneratingFunctionMask; + + ProjectionMask() { + super(3); + } + + /** + * A table-generating function transforms a single input row to multiple output rows. For example, explode(array('A','B','C') will produce 3 one-column rows, which are row1: 'A'; row2: 'B'; row3: 'C'. + * + */ + public LateralView.ProjectionMask withTableGeneratingFunction(Function nestedMask) { + _tableGeneratingFunctionMask = nestedMask.apply(((_tableGeneratingFunctionMask == null)?LateralView.TableGeneratingFunction.createMask():_tableGeneratingFunctionMask)); + getDataMap().put("tableGeneratingFunction", _tableGeneratingFunctionMask.getDataMap()); + return this; + } + + /** + * A table-generating function transforms a single input row to multiple output rows. For example, explode(array('A','B','C') will produce 3 one-column rows, which are row1: 'A'; row2: 'B'; row3: 'C'. + * + */ + public LateralView.ProjectionMask withTableGeneratingFunction() { + _tableGeneratingFunctionMask = null; + getDataMap().put("tableGeneratingFunction", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Represents the alias for referencing the generated virtual table. It will be used in subsequent statements (eg. filter, groupBy) in the sliding window feature definition. + * + */ + public LateralView.ProjectionMask withVirtualTableAlias() { + getDataMap().put("virtualTableAlias", MaskMap.POSITIVE_MASK); + return this; + } + + } + + @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\LateralView.pdl.") + public static class TableGeneratingFunction + extends UnionTemplate + { + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}}]", SchemaFormatType.PDL)); + private com.linkedin.feathr.compute.SqlExpression _sqlExpressionMember = null; + private LateralView.TableGeneratingFunction.ChangeListener __changeListener = new LateralView.TableGeneratingFunction.ChangeListener(this); + private final static DataSchema MEMBER_SqlExpression = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.SqlExpression"); + + public TableGeneratingFunction() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public TableGeneratingFunction(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static LateralView.TableGeneratingFunction create(com.linkedin.feathr.compute.SqlExpression value) { + LateralView.TableGeneratingFunction newUnion = new LateralView.TableGeneratingFunction(); + newUnion.setSqlExpression(value); + return newUnion; + } + + public boolean isSqlExpression() { + return memberIs("com.linkedin.feathr.compute.SqlExpression"); + } + + public com.linkedin.feathr.compute.SqlExpression getSqlExpression() { + checkNotNull(); + if (_sqlExpressionMember!= null) { + return _sqlExpressionMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.SqlExpression"); + _sqlExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.SqlExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _sqlExpressionMember; + } + + public void setSqlExpression(com.linkedin.feathr.compute.SqlExpression value) { + checkNotNull(); + super._map.clear(); + _sqlExpressionMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.SqlExpression", value.data()); + } + + public static LateralView.TableGeneratingFunction.ProjectionMask createMask() { + return new LateralView.TableGeneratingFunction.ProjectionMask(); + } + + @Override + public LateralView.TableGeneratingFunction clone() + throws CloneNotSupportedException + { + LateralView.TableGeneratingFunction __clone = ((LateralView.TableGeneratingFunction) super.clone()); + __clone.__changeListener = new LateralView.TableGeneratingFunction.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public LateralView.TableGeneratingFunction copy() + throws CloneNotSupportedException + { + LateralView.TableGeneratingFunction __copy = ((LateralView.TableGeneratingFunction) super.copy()); + __copy._sqlExpressionMember = null; + __copy.__changeListener = new LateralView.TableGeneratingFunction.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final LateralView.TableGeneratingFunction __objectRef; + + private ChangeListener(LateralView.TableGeneratingFunction reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "com.linkedin.feathr.compute.SqlExpression": + __objectRef._sqlExpressionMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.compute.SqlExpression.Fields SqlExpression() { + return new com.linkedin.feathr.compute.SqlExpression.Fields(getPathComponents(), "com.linkedin.feathr.compute.SqlExpression"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.SqlExpression.ProjectionMask _SqlExpressionMask; + + ProjectionMask() { + super(2); + } + + public LateralView.TableGeneratingFunction.ProjectionMask withSqlExpression(Function nestedMask) { + _SqlExpressionMask = nestedMask.apply(((_SqlExpressionMask == null)?com.linkedin.feathr.compute.SqlExpression.createMask():_SqlExpressionMask)); + getDataMap().put("com.linkedin.feathr.compute.SqlExpression", _SqlExpressionMask.getDataMap()); + return this; + } + + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralViewArray.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralViewArray.java new file mode 100644 index 000000000..8a4260c3c --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralViewArray.java @@ -0,0 +1,118 @@ + +package com.linkedin.feathr.compute; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.schema.ArrayDataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.TemplateOutputCastException; +import com.linkedin.data.template.WrappingArrayTemplate; + +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SlidingWindowFeature.pdl.") +public class LateralViewArray + extends WrappingArrayTemplate +{ + + private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[{namespace com.linkedin.feathr.compute/**Lateral view is used in conjunction with table generating functions (eg. the most commonly used explode()), which typically generates zero or more output rows for each input row. A lateral view first applies the table generating function to each row of base table, and then joins resulting output rows to the input rows to form a virtual table with the supplied table alias. For more details and examples, refer to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+LateralView.*/record LateralView{/**A table-generating function transforms a single input row to multiple output rows. For example, explode(array('A','B','C') will produce 3 one-column rows, which are row1: 'A'; row2: 'B'; row3: 'C'.*/tableGeneratingFunction:union[/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}]/**Represents the alias for referencing the generated virtual table. It will be used in subsequent statements (eg. filter, groupBy) in the sliding window feature definition.*/virtualTableAlias:string}}]", SchemaFormatType.PDL)); + + public LateralViewArray() { + this(new DataList()); + } + + public LateralViewArray(int initialCapacity) { + this(new DataList(initialCapacity)); + } + + public LateralViewArray(Collection c) { + this(new DataList(c.size())); + addAll(c); + } + + public LateralViewArray(DataList data) { + super(data, SCHEMA, LateralView.class); + } + + public LateralViewArray(LateralView first, LateralView... rest) { + this(new DataList((rest.length + 1))); + add(first); + addAll(Arrays.asList(rest)); + } + + public static ArrayDataSchema dataSchema() { + return SCHEMA; + } + + public static LateralViewArray.ProjectionMask createMask() { + return new LateralViewArray.ProjectionMask(); + } + + @Override + public LateralViewArray clone() + throws CloneNotSupportedException + { + LateralViewArray __clone = ((LateralViewArray) super.clone()); + return __clone; + } + + @Override + public LateralViewArray copy() + throws CloneNotSupportedException + { + LateralViewArray __copy = ((LateralViewArray) super.copy()); + return __copy; + } + + @Override + protected LateralView coerceOutput(Object object) + throws TemplateOutputCastException + { + assert(object != null); + return ((object == null)?null:new LateralView(DataTemplateUtil.castOrThrow(object, DataMap.class))); + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.compute.LateralView.Fields items() { + return new com.linkedin.feathr.compute.LateralView.Fields(getPathComponents(), PathSpec.WILDCARD); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.LateralView.ProjectionMask _itemsMask; + + ProjectionMask() { + super(4); + } + + public LateralViewArray.ProjectionMask withItems(Function nestedMask) { + _itemsMask = nestedMask.apply(((_itemsMask == null)?LateralView.createMask():_itemsMask)); + getDataMap().put("$*", _itemsMask.getDataMap()); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Lookup.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Lookup.java new file mode 100644 index 000000000..85dba2932 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Lookup.java @@ -0,0 +1,1587 @@ + +package com.linkedin.feathr.compute; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.ArrayDataSchema; +import com.linkedin.data.schema.DataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.UnionDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.TemplateOutputCastException; +import com.linkedin.data.template.UnionTemplate; +import com.linkedin.data.template.WrappingArrayTemplate; + + +/** + * A node to represent a feature which is to be computed by using an already computed feature as the key. + * https://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?spaceKey=ENGS&title=feathr+Offline+User+Guide#FrameOfflineUserGuide-sequentialjoin + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Lookup.pdl.") +public class Lookup + extends RecordTemplate +{ + + private final static Lookup.Fields _fields = new Lookup.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**A node to represent a feature which is to be computed by using an already computed feature as the key.\r\nhttps://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?spaceKey=ENGS&title=feathr+Offline+User+Guide#FrameOfflineUserGuide-sequentialjoin*/record Lookup includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**An array of references to a node and keys.\r\n\r\nFor now, we do not support lookup of just a key reference, but we have added that as a placeholder.\r\n\r\nA node reference consists of node id and a key reference.\r\nIn sequential join the lookup key would be a combination of the\r\nfeature node representing the base feature (lookup node) and the key associated with it. For example,:-\r\nseqJoinFeature: {\r\n base: {key: x, feature: baseFeature}\r\n expansion: {key: y, feature: expansionFeature}\r\n aggregation: UNION\r\n}\r\nHere, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would\r\npoint to the index of \"x\" in the key array of baseFeature.*/lookupKey:array[union[/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}KeyReference]]/**The node id of the node containing the expansion feature.*/lookupNode:NodeId/**Aggregation type as listed in\r\nhttps://jarvis.corp.linkedin.com/codesearch/result/\r\n?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7\r\n*/aggregation:string/**feature name of the feature which would be computed.\r\nwe need feature name here for 2 main reasons.\r\n1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and\r\nwe want to leverage that.\r\n2. For default values. Similar to above, there are existing APIs which create default value map from feature name ->\r\ndefault value.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}", SchemaFormatType.PDL)); + private Integer _idField = null; + private ConcreteKey _concreteKeyField = null; + private Lookup.LookupKeyArray _lookupKeyField = null; + private Integer _lookupNodeField = null; + private String _aggregationField = null; + private String _featureNameField = null; + private FeatureVersion _featureVersionField = null; + private Lookup.ChangeListener __changeListener = new Lookup.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); + private final static RecordDataSchema.Field FIELD_ConcreteKey = SCHEMA.getField("concreteKey"); + private final static RecordDataSchema.Field FIELD_LookupKey = SCHEMA.getField("lookupKey"); + private final static RecordDataSchema.Field FIELD_LookupNode = SCHEMA.getField("lookupNode"); + private final static RecordDataSchema.Field FIELD_Aggregation = SCHEMA.getField("aggregation"); + private final static RecordDataSchema.Field FIELD_FeatureName = SCHEMA.getField("featureName"); + private final static RecordDataSchema.Field FIELD_FeatureVersion = SCHEMA.getField("featureVersion"); + + public Lookup() { + super(new DataMap(10, 0.75F), SCHEMA, 4); + addChangeListener(__changeListener); + } + + public Lookup(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static Lookup.Fields fields() { + return _fields; + } + + public static Lookup.ProjectionMask createMask() { + return new Lookup.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for id + * + * @see Lookup.Fields#id + */ + public boolean hasId() { + if (_idField!= null) { + return true; + } + return super._map.containsKey("id"); + } + + /** + * Remover for id + * + * @see Lookup.Fields#id + */ + public void removeId() { + super._map.remove("id"); + } + + /** + * Getter for id + * + * @see Lookup.Fields#id + */ + public Integer getId(GetMode mode) { + switch (mode) { + case STRICT: + return getId(); + case DEFAULT: + case NULL: + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for id + * + * @return + * Required field. Could be null for partial record. + * @see Lookup.Fields#id + */ + @Nonnull + public Integer getId() { + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("id"); + } + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + + /** + * Setter for id + * + * @see Lookup.Fields#id + */ + public Lookup setId(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setId(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.Lookup"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeId(); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + } + return this; + } + + /** + * Setter for id + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Lookup.Fields#id + */ + public Lookup setId( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.Lookup to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + return this; + } + + /** + * Setter for id + * + * @see Lookup.Fields#id + */ + public Lookup setId(int value) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + return this; + } + + /** + * Existence checker for concreteKey + * + * @see Lookup.Fields#concreteKey + */ + public boolean hasConcreteKey() { + if (_concreteKeyField!= null) { + return true; + } + return super._map.containsKey("concreteKey"); + } + + /** + * Remover for concreteKey + * + * @see Lookup.Fields#concreteKey + */ + public void removeConcreteKey() { + super._map.remove("concreteKey"); + } + + /** + * Getter for concreteKey + * + * @see Lookup.Fields#concreteKey + */ + public ConcreteKey getConcreteKey(GetMode mode) { + return getConcreteKey(); + } + + /** + * Getter for concreteKey + * + * @return + * Optional field. Always check for null. + * @see Lookup.Fields#concreteKey + */ + @Nullable + public ConcreteKey getConcreteKey() { + if (_concreteKeyField!= null) { + return _concreteKeyField; + } else { + Object __rawValue = super._map.get("concreteKey"); + _concreteKeyField = ((__rawValue == null)?null:new ConcreteKey(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _concreteKeyField; + } + } + + /** + * Setter for concreteKey + * + * @see Lookup.Fields#concreteKey + */ + public Lookup setConcreteKey(ConcreteKey value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setConcreteKey(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeConcreteKey(); + } else { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + break; + } + return this; + } + + /** + * Setter for concreteKey + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Lookup.Fields#concreteKey + */ + public Lookup setConcreteKey( + @Nonnull + ConcreteKey value) { + if (value == null) { + throw new NullPointerException("Cannot set field concreteKey of com.linkedin.feathr.compute.Lookup to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + return this; + } + + /** + * Existence checker for lookupKey + * + * @see Lookup.Fields#lookupKey + */ + public boolean hasLookupKey() { + if (_lookupKeyField!= null) { + return true; + } + return super._map.containsKey("lookupKey"); + } + + /** + * Remover for lookupKey + * + * @see Lookup.Fields#lookupKey + */ + public void removeLookupKey() { + super._map.remove("lookupKey"); + } + + /** + * Getter for lookupKey + * + * @see Lookup.Fields#lookupKey + */ + public Lookup.LookupKeyArray getLookupKey(GetMode mode) { + switch (mode) { + case STRICT: + return getLookupKey(); + case DEFAULT: + case NULL: + if (_lookupKeyField!= null) { + return _lookupKeyField; + } else { + Object __rawValue = super._map.get("lookupKey"); + _lookupKeyField = ((__rawValue == null)?null:new Lookup.LookupKeyArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _lookupKeyField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for lookupKey + * + * @return + * Required field. Could be null for partial record. + * @see Lookup.Fields#lookupKey + */ + @Nonnull + public Lookup.LookupKeyArray getLookupKey() { + if (_lookupKeyField!= null) { + return _lookupKeyField; + } else { + Object __rawValue = super._map.get("lookupKey"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("lookupKey"); + } + _lookupKeyField = ((__rawValue == null)?null:new Lookup.LookupKeyArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _lookupKeyField; + } + } + + /** + * Setter for lookupKey + * + * @see Lookup.Fields#lookupKey + */ + public Lookup setLookupKey(Lookup.LookupKeyArray value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setLookupKey(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field lookupKey of com.linkedin.feathr.compute.Lookup"); + } else { + CheckedUtil.putWithoutChecking(super._map, "lookupKey", value.data()); + _lookupKeyField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeLookupKey(); + } else { + CheckedUtil.putWithoutChecking(super._map, "lookupKey", value.data()); + _lookupKeyField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "lookupKey", value.data()); + _lookupKeyField = value; + } + break; + } + return this; + } + + /** + * Setter for lookupKey + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Lookup.Fields#lookupKey + */ + public Lookup setLookupKey( + @Nonnull + Lookup.LookupKeyArray value) { + if (value == null) { + throw new NullPointerException("Cannot set field lookupKey of com.linkedin.feathr.compute.Lookup to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "lookupKey", value.data()); + _lookupKeyField = value; + } + return this; + } + + /** + * Existence checker for lookupNode + * + * @see Lookup.Fields#lookupNode + */ + public boolean hasLookupNode() { + if (_lookupNodeField!= null) { + return true; + } + return super._map.containsKey("lookupNode"); + } + + /** + * Remover for lookupNode + * + * @see Lookup.Fields#lookupNode + */ + public void removeLookupNode() { + super._map.remove("lookupNode"); + } + + /** + * Getter for lookupNode + * + * @see Lookup.Fields#lookupNode + */ + public Integer getLookupNode(GetMode mode) { + switch (mode) { + case STRICT: + return getLookupNode(); + case DEFAULT: + case NULL: + if (_lookupNodeField!= null) { + return _lookupNodeField; + } else { + Object __rawValue = super._map.get("lookupNode"); + _lookupNodeField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _lookupNodeField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for lookupNode + * + * @return + * Required field. Could be null for partial record. + * @see Lookup.Fields#lookupNode + */ + @Nonnull + public Integer getLookupNode() { + if (_lookupNodeField!= null) { + return _lookupNodeField; + } else { + Object __rawValue = super._map.get("lookupNode"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("lookupNode"); + } + _lookupNodeField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _lookupNodeField; + } + } + + /** + * Setter for lookupNode + * + * @see Lookup.Fields#lookupNode + */ + public Lookup setLookupNode(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setLookupNode(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field lookupNode of com.linkedin.feathr.compute.Lookup"); + } else { + CheckedUtil.putWithoutChecking(super._map, "lookupNode", DataTemplateUtil.coerceIntInput(value)); + _lookupNodeField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeLookupNode(); + } else { + CheckedUtil.putWithoutChecking(super._map, "lookupNode", DataTemplateUtil.coerceIntInput(value)); + _lookupNodeField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "lookupNode", DataTemplateUtil.coerceIntInput(value)); + _lookupNodeField = value; + } + break; + } + return this; + } + + /** + * Setter for lookupNode + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Lookup.Fields#lookupNode + */ + public Lookup setLookupNode( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field lookupNode of com.linkedin.feathr.compute.Lookup to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "lookupNode", DataTemplateUtil.coerceIntInput(value)); + _lookupNodeField = value; + } + return this; + } + + /** + * Setter for lookupNode + * + * @see Lookup.Fields#lookupNode + */ + public Lookup setLookupNode(int value) { + CheckedUtil.putWithoutChecking(super._map, "lookupNode", DataTemplateUtil.coerceIntInput(value)); + _lookupNodeField = value; + return this; + } + + /** + * Existence checker for aggregation + * + * @see Lookup.Fields#aggregation + */ + public boolean hasAggregation() { + if (_aggregationField!= null) { + return true; + } + return super._map.containsKey("aggregation"); + } + + /** + * Remover for aggregation + * + * @see Lookup.Fields#aggregation + */ + public void removeAggregation() { + super._map.remove("aggregation"); + } + + /** + * Getter for aggregation + * + * @see Lookup.Fields#aggregation + */ + public String getAggregation(GetMode mode) { + switch (mode) { + case STRICT: + return getAggregation(); + case DEFAULT: + case NULL: + if (_aggregationField!= null) { + return _aggregationField; + } else { + Object __rawValue = super._map.get("aggregation"); + _aggregationField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _aggregationField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for aggregation + * + * @return + * Required field. Could be null for partial record. + * @see Lookup.Fields#aggregation + */ + @Nonnull + public String getAggregation() { + if (_aggregationField!= null) { + return _aggregationField; + } else { + Object __rawValue = super._map.get("aggregation"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("aggregation"); + } + _aggregationField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _aggregationField; + } + } + + /** + * Setter for aggregation + * + * @see Lookup.Fields#aggregation + */ + public Lookup setAggregation(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setAggregation(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field aggregation of com.linkedin.feathr.compute.Lookup"); + } else { + CheckedUtil.putWithoutChecking(super._map, "aggregation", value); + _aggregationField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeAggregation(); + } else { + CheckedUtil.putWithoutChecking(super._map, "aggregation", value); + _aggregationField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "aggregation", value); + _aggregationField = value; + } + break; + } + return this; + } + + /** + * Setter for aggregation + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Lookup.Fields#aggregation + */ + public Lookup setAggregation( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field aggregation of com.linkedin.feathr.compute.Lookup to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "aggregation", value); + _aggregationField = value; + } + return this; + } + + /** + * Existence checker for featureName + * + * @see Lookup.Fields#featureName + */ + public boolean hasFeatureName() { + if (_featureNameField!= null) { + return true; + } + return super._map.containsKey("featureName"); + } + + /** + * Remover for featureName + * + * @see Lookup.Fields#featureName + */ + public void removeFeatureName() { + super._map.remove("featureName"); + } + + /** + * Getter for featureName + * + * @see Lookup.Fields#featureName + */ + public String getFeatureName(GetMode mode) { + switch (mode) { + case STRICT: + return getFeatureName(); + case DEFAULT: + case NULL: + if (_featureNameField!= null) { + return _featureNameField; + } else { + Object __rawValue = super._map.get("featureName"); + _featureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _featureNameField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for featureName + * + * @return + * Required field. Could be null for partial record. + * @see Lookup.Fields#featureName + */ + @Nonnull + public String getFeatureName() { + if (_featureNameField!= null) { + return _featureNameField; + } else { + Object __rawValue = super._map.get("featureName"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("featureName"); + } + _featureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _featureNameField; + } + } + + /** + * Setter for featureName + * + * @see Lookup.Fields#featureName + */ + public Lookup setFeatureName(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFeatureName(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field featureName of com.linkedin.feathr.compute.Lookup"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureName", value); + _featureNameField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeFeatureName(); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureName", value); + _featureNameField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "featureName", value); + _featureNameField = value; + } + break; + } + return this; + } + + /** + * Setter for featureName + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Lookup.Fields#featureName + */ + public Lookup setFeatureName( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field featureName of com.linkedin.feathr.compute.Lookup to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureName", value); + _featureNameField = value; + } + return this; + } + + /** + * Existence checker for featureVersion + * + * @see Lookup.Fields#featureVersion + */ + public boolean hasFeatureVersion() { + if (_featureVersionField!= null) { + return true; + } + return super._map.containsKey("featureVersion"); + } + + /** + * Remover for featureVersion + * + * @see Lookup.Fields#featureVersion + */ + public void removeFeatureVersion() { + super._map.remove("featureVersion"); + } + + /** + * Getter for featureVersion + * + * @see Lookup.Fields#featureVersion + */ + public FeatureVersion getFeatureVersion(GetMode mode) { + switch (mode) { + case STRICT: + return getFeatureVersion(); + case DEFAULT: + case NULL: + if (_featureVersionField!= null) { + return _featureVersionField; + } else { + Object __rawValue = super._map.get("featureVersion"); + _featureVersionField = ((__rawValue == null)?null:new FeatureVersion(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _featureVersionField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for featureVersion + * + * @return + * Required field. Could be null for partial record. + * @see Lookup.Fields#featureVersion + */ + @Nonnull + public FeatureVersion getFeatureVersion() { + if (_featureVersionField!= null) { + return _featureVersionField; + } else { + Object __rawValue = super._map.get("featureVersion"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("featureVersion"); + } + _featureVersionField = ((__rawValue == null)?null:new FeatureVersion(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _featureVersionField; + } + } + + /** + * Setter for featureVersion + * + * @see Lookup.Fields#featureVersion + */ + public Lookup setFeatureVersion(FeatureVersion value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFeatureVersion(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field featureVersion of com.linkedin.feathr.compute.Lookup"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); + _featureVersionField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeFeatureVersion(); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); + _featureVersionField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); + _featureVersionField = value; + } + break; + } + return this; + } + + /** + * Setter for featureVersion + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Lookup.Fields#featureVersion + */ + public Lookup setFeatureVersion( + @Nonnull + FeatureVersion value) { + if (value == null) { + throw new NullPointerException("Cannot set field featureVersion of com.linkedin.feathr.compute.Lookup to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); + _featureVersionField = value; + } + return this; + } + + @Override + public Lookup clone() + throws CloneNotSupportedException + { + Lookup __clone = ((Lookup) super.clone()); + __clone.__changeListener = new Lookup.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public Lookup copy() + throws CloneNotSupportedException + { + Lookup __copy = ((Lookup) super.copy()); + __copy._lookupKeyField = null; + __copy._featureNameField = null; + __copy._lookupNodeField = null; + __copy._aggregationField = null; + __copy._idField = null; + __copy._concreteKeyField = null; + __copy._featureVersionField = null; + __copy.__changeListener = new Lookup.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final Lookup __objectRef; + + private ChangeListener(Lookup reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "lookupKey": + __objectRef._lookupKeyField = null; + break; + case "featureName": + __objectRef._featureNameField = null; + break; + case "lookupNode": + __objectRef._lookupNodeField = null; + break; + case "aggregation": + __objectRef._aggregationField = null; + break; + case "id": + __objectRef._idField = null; + break; + case "concreteKey": + __objectRef._concreteKeyField = null; + break; + case "featureVersion": + __objectRef._featureVersionField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The node would be represented by this id. + * + */ + public PathSpec id() { + return new PathSpec(getPathComponents(), "id"); + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public com.linkedin.feathr.compute.ConcreteKey.Fields concreteKey() { + return new com.linkedin.feathr.compute.ConcreteKey.Fields(getPathComponents(), "concreteKey"); + } + + /** + * An array of references to a node and keys. + * + * For now, we do not support lookup of just a key reference, but we have added that as a placeholder. + * + * A node reference consists of node id and a key reference. + * In sequential join the lookup key would be a combination of the + * feature node representing the base feature (lookup node) and the key associated with it. For example,:- + * seqJoinFeature: { + * base: {key: x, feature: baseFeature} + * expansion: {key: y, feature: expansionFeature} + * aggregation: UNION + * } + * Here, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would + * point to the index of "x" in the key array of baseFeature. + * + */ + public com.linkedin.feathr.compute.Lookup.LookupKeyArray.Fields lookupKey() { + return new com.linkedin.feathr.compute.Lookup.LookupKeyArray.Fields(getPathComponents(), "lookupKey"); + } + + /** + * An array of references to a node and keys. + * + * For now, we do not support lookup of just a key reference, but we have added that as a placeholder. + * + * A node reference consists of node id and a key reference. + * In sequential join the lookup key would be a combination of the + * feature node representing the base feature (lookup node) and the key associated with it. For example,:- + * seqJoinFeature: { + * base: {key: x, feature: baseFeature} + * expansion: {key: y, feature: expansionFeature} + * aggregation: UNION + * } + * Here, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would + * point to the index of "x" in the key array of baseFeature. + * + */ + public PathSpec lookupKey(Integer start, Integer count) { + PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "lookupKey"); + if (start!= null) { + arrayPathSpec.setAttribute("start", start); + } + if (count!= null) { + arrayPathSpec.setAttribute("count", count); + } + return arrayPathSpec; + } + + /** + * The node id of the node containing the expansion feature. + * + */ + public PathSpec lookupNode() { + return new PathSpec(getPathComponents(), "lookupNode"); + } + + /** + * Aggregation type as listed in + * https://jarvis.corp.linkedin.com/codesearch/result/ + * ?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7 + * + * + */ + public PathSpec aggregation() { + return new PathSpec(getPathComponents(), "aggregation"); + } + + /** + * feature name of the feature which would be computed. + * we need feature name here for 2 main reasons. + * 1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and + * we want to leverage that. + * 2. For default values. Similar to above, there are existing APIs which create default value map from feature name -> + * default value. + * + */ + public PathSpec featureName() { + return new PathSpec(getPathComponents(), "featureName"); + } + + /** + * feature version of the feature + * + */ + public com.linkedin.feathr.compute.FeatureVersion.Fields featureVersion() { + return new com.linkedin.feathr.compute.FeatureVersion.Fields(getPathComponents(), "featureVersion"); + } + + } + + @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Lookup.pdl.") + public static class LookupKey + extends UnionTemplate + { + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}}com.linkedin.feathr.compute.KeyReference]", SchemaFormatType.PDL)); + private com.linkedin.feathr.compute.NodeReference _nodeReferenceMember = null; + private com.linkedin.feathr.compute.KeyReference _keyReferenceMember = null; + private Lookup.LookupKey.ChangeListener __changeListener = new Lookup.LookupKey.ChangeListener(this); + private final static DataSchema MEMBER_NodeReference = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.NodeReference"); + private final static DataSchema MEMBER_KeyReference = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.KeyReference"); + + public LookupKey() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public LookupKey(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static Lookup.LookupKey create(com.linkedin.feathr.compute.NodeReference value) { + Lookup.LookupKey newUnion = new Lookup.LookupKey(); + newUnion.setNodeReference(value); + return newUnion; + } + + public boolean isNodeReference() { + return memberIs("com.linkedin.feathr.compute.NodeReference"); + } + + public com.linkedin.feathr.compute.NodeReference getNodeReference() { + checkNotNull(); + if (_nodeReferenceMember!= null) { + return _nodeReferenceMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.NodeReference"); + _nodeReferenceMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.NodeReference(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _nodeReferenceMember; + } + + public void setNodeReference(com.linkedin.feathr.compute.NodeReference value) { + checkNotNull(); + super._map.clear(); + _nodeReferenceMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.NodeReference", value.data()); + } + + public static Lookup.LookupKey create(com.linkedin.feathr.compute.KeyReference value) { + Lookup.LookupKey newUnion = new Lookup.LookupKey(); + newUnion.setKeyReference(value); + return newUnion; + } + + public boolean isKeyReference() { + return memberIs("com.linkedin.feathr.compute.KeyReference"); + } + + public com.linkedin.feathr.compute.KeyReference getKeyReference() { + checkNotNull(); + if (_keyReferenceMember!= null) { + return _keyReferenceMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.KeyReference"); + _keyReferenceMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.KeyReference(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _keyReferenceMember; + } + + public void setKeyReference(com.linkedin.feathr.compute.KeyReference value) { + checkNotNull(); + super._map.clear(); + _keyReferenceMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.KeyReference", value.data()); + } + + public static Lookup.LookupKey.ProjectionMask createMask() { + return new Lookup.LookupKey.ProjectionMask(); + } + + @Override + public Lookup.LookupKey clone() + throws CloneNotSupportedException + { + Lookup.LookupKey __clone = ((Lookup.LookupKey) super.clone()); + __clone.__changeListener = new Lookup.LookupKey.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public Lookup.LookupKey copy() + throws CloneNotSupportedException + { + Lookup.LookupKey __copy = ((Lookup.LookupKey) super.copy()); + __copy._keyReferenceMember = null; + __copy._nodeReferenceMember = null; + __copy.__changeListener = new Lookup.LookupKey.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final Lookup.LookupKey __objectRef; + + private ChangeListener(Lookup.LookupKey reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "com.linkedin.feathr.compute.KeyReference": + __objectRef._keyReferenceMember = null; + break; + case "com.linkedin.feathr.compute.NodeReference": + __objectRef._nodeReferenceMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.compute.NodeReference.Fields NodeReference() { + return new com.linkedin.feathr.compute.NodeReference.Fields(getPathComponents(), "com.linkedin.feathr.compute.NodeReference"); + } + + public com.linkedin.feathr.compute.KeyReference.Fields KeyReference() { + return new com.linkedin.feathr.compute.KeyReference.Fields(getPathComponents(), "com.linkedin.feathr.compute.KeyReference"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.NodeReference.ProjectionMask _NodeReferenceMask; + private com.linkedin.feathr.compute.KeyReference.ProjectionMask _KeyReferenceMask; + + ProjectionMask() { + super(3); + } + + public Lookup.LookupKey.ProjectionMask withNodeReference(Function nestedMask) { + _NodeReferenceMask = nestedMask.apply(((_NodeReferenceMask == null)?com.linkedin.feathr.compute.NodeReference.createMask():_NodeReferenceMask)); + getDataMap().put("com.linkedin.feathr.compute.NodeReference", _NodeReferenceMask.getDataMap()); + return this; + } + + public Lookup.LookupKey.ProjectionMask withKeyReference(Function nestedMask) { + _KeyReferenceMask = nestedMask.apply(((_KeyReferenceMask == null)?com.linkedin.feathr.compute.KeyReference.createMask():_KeyReferenceMask)); + getDataMap().put("com.linkedin.feathr.compute.KeyReference", _KeyReferenceMask.getDataMap()); + return this; + } + + } + + } + + @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Lookup.pdl.") + public static class LookupKeyArray + extends WrappingArrayTemplate + { + + private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[union[{namespace com.linkedin.feathr.compute/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}}com.linkedin.feathr.compute.KeyReference]]", SchemaFormatType.PDL)); + + public LookupKeyArray() { + this(new DataList()); + } + + public LookupKeyArray(int initialCapacity) { + this(new DataList(initialCapacity)); + } + + public LookupKeyArray(Collection c) { + this(new DataList(c.size())); + addAll(c); + } + + public LookupKeyArray(DataList data) { + super(data, SCHEMA, Lookup.LookupKey.class); + } + + public LookupKeyArray(Lookup.LookupKey first, Lookup.LookupKey... rest) { + this(new DataList((rest.length + 1))); + add(first); + addAll(Arrays.asList(rest)); + } + + public static ArrayDataSchema dataSchema() { + return SCHEMA; + } + + public static Lookup.LookupKeyArray.ProjectionMask createMask() { + return new Lookup.LookupKeyArray.ProjectionMask(); + } + + @Override + public Lookup.LookupKeyArray clone() + throws CloneNotSupportedException + { + Lookup.LookupKeyArray __clone = ((Lookup.LookupKeyArray) super.clone()); + return __clone; + } + + @Override + public Lookup.LookupKeyArray copy() + throws CloneNotSupportedException + { + Lookup.LookupKeyArray __copy = ((Lookup.LookupKeyArray) super.copy()); + return __copy; + } + + @Override + protected Lookup.LookupKey coerceOutput(Object object) + throws TemplateOutputCastException + { + assert(object != null); + return ((object == null)?null:new Lookup.LookupKey(object)); + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.compute.Lookup.LookupKey.Fields items() { + return new com.linkedin.feathr.compute.Lookup.LookupKey.Fields(getPathComponents(), PathSpec.WILDCARD); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.Lookup.LookupKey.ProjectionMask _itemsMask; + + ProjectionMask() { + super(4); + } + + public Lookup.LookupKeyArray.ProjectionMask withItems(Function nestedMask) { + _itemsMask = nestedMask.apply(((_itemsMask == null)?Lookup.LookupKey.createMask():_itemsMask)); + getDataMap().put("$*", _itemsMask.getDataMap()); + return this; + } + + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.ConcreteKey.ProjectionMask _concreteKeyMask; + private com.linkedin.feathr.compute.Lookup.LookupKeyArray.ProjectionMask _lookupKeyMask; + private com.linkedin.feathr.compute.FeatureVersion.ProjectionMask _featureVersionMask; + + ProjectionMask() { + super(10); + } + + /** + * The node would be represented by this id. + * + */ + public Lookup.ProjectionMask withId() { + getDataMap().put("id", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public Lookup.ProjectionMask withConcreteKey(Function nestedMask) { + _concreteKeyMask = nestedMask.apply(((_concreteKeyMask == null)?ConcreteKey.createMask():_concreteKeyMask)); + getDataMap().put("concreteKey", _concreteKeyMask.getDataMap()); + return this; + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public Lookup.ProjectionMask withConcreteKey() { + _concreteKeyMask = null; + getDataMap().put("concreteKey", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * An array of references to a node and keys. + * + * For now, we do not support lookup of just a key reference, but we have added that as a placeholder. + * + * A node reference consists of node id and a key reference. + * In sequential join the lookup key would be a combination of the + * feature node representing the base feature (lookup node) and the key associated with it. For example,:- + * seqJoinFeature: { + * base: {key: x, feature: baseFeature} + * expansion: {key: y, feature: expansionFeature} + * aggregation: UNION + * } + * Here, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would + * point to the index of "x" in the key array of baseFeature. + * + */ + public Lookup.ProjectionMask withLookupKey(Function nestedMask) { + _lookupKeyMask = nestedMask.apply(((_lookupKeyMask == null)?Lookup.LookupKeyArray.createMask():_lookupKeyMask)); + getDataMap().put("lookupKey", _lookupKeyMask.getDataMap()); + return this; + } + + /** + * An array of references to a node and keys. + * + * For now, we do not support lookup of just a key reference, but we have added that as a placeholder. + * + * A node reference consists of node id and a key reference. + * In sequential join the lookup key would be a combination of the + * feature node representing the base feature (lookup node) and the key associated with it. For example,:- + * seqJoinFeature: { + * base: {key: x, feature: baseFeature} + * expansion: {key: y, feature: expansionFeature} + * aggregation: UNION + * } + * Here, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would + * point to the index of "x" in the key array of baseFeature. + * + */ + public Lookup.ProjectionMask withLookupKey() { + _lookupKeyMask = null; + getDataMap().put("lookupKey", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * An array of references to a node and keys. + * + * For now, we do not support lookup of just a key reference, but we have added that as a placeholder. + * + * A node reference consists of node id and a key reference. + * In sequential join the lookup key would be a combination of the + * feature node representing the base feature (lookup node) and the key associated with it. For example,:- + * seqJoinFeature: { + * base: {key: x, feature: baseFeature} + * expansion: {key: y, feature: expansionFeature} + * aggregation: UNION + * } + * Here, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would + * point to the index of "x" in the key array of baseFeature. + * + */ + public Lookup.ProjectionMask withLookupKey(Function nestedMask, Integer start, Integer count) { + _lookupKeyMask = nestedMask.apply(((_lookupKeyMask == null)?Lookup.LookupKeyArray.createMask():_lookupKeyMask)); + getDataMap().put("lookupKey", _lookupKeyMask.getDataMap()); + if (start!= null) { + getDataMap().getDataMap("lookupKey").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("lookupKey").put("$count", count); + } + return this; + } + + /** + * An array of references to a node and keys. + * + * For now, we do not support lookup of just a key reference, but we have added that as a placeholder. + * + * A node reference consists of node id and a key reference. + * In sequential join the lookup key would be a combination of the + * feature node representing the base feature (lookup node) and the key associated with it. For example,:- + * seqJoinFeature: { + * base: {key: x, feature: baseFeature} + * expansion: {key: y, feature: expansionFeature} + * aggregation: UNION + * } + * Here, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would + * point to the index of "x" in the key array of baseFeature. + * + */ + public Lookup.ProjectionMask withLookupKey(Integer start, Integer count) { + _lookupKeyMask = null; + getDataMap().put("lookupKey", new DataMap(3)); + if (start!= null) { + getDataMap().getDataMap("lookupKey").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("lookupKey").put("$count", count); + } + return this; + } + + /** + * The node id of the node containing the expansion feature. + * + */ + public Lookup.ProjectionMask withLookupNode() { + getDataMap().put("lookupNode", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Aggregation type as listed in + * https://jarvis.corp.linkedin.com/codesearch/result/ + * ?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7 + * + * + */ + public Lookup.ProjectionMask withAggregation() { + getDataMap().put("aggregation", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * feature name of the feature which would be computed. + * we need feature name here for 2 main reasons. + * 1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and + * we want to leverage that. + * 2. For default values. Similar to above, there are existing APIs which create default value map from feature name -> + * default value. + * + */ + public Lookup.ProjectionMask withFeatureName() { + getDataMap().put("featureName", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * feature version of the feature + * + */ + public Lookup.ProjectionMask withFeatureVersion(Function nestedMask) { + _featureVersionMask = nestedMask.apply(((_featureVersionMask == null)?FeatureVersion.createMask():_featureVersionMask)); + getDataMap().put("featureVersion", _featureVersionMask.getDataMap()); + return this; + } + + /** + * feature version of the feature + * + */ + public Lookup.ProjectionMask withFeatureVersion() { + _featureVersionMask = null; + getDataMap().put("featureVersion", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/MvelExpression.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/MvelExpression.java new file mode 100644 index 000000000..e26a4db0e --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/MvelExpression.java @@ -0,0 +1,260 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * An expression in MVEL language. For more information please refer to go/framemvel. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\MvelExpression.pdl.") +public class MvelExpression + extends RecordTemplate +{ + + private final static MvelExpression.Fields _fields = new MvelExpression.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**An expression in MVEL language. For more information please refer to go/framemvel.*/record MvelExpression{/**The MVEL expression.*/mvel:string}", SchemaFormatType.PDL)); + private String _mvelField = null; + private MvelExpression.ChangeListener __changeListener = new MvelExpression.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Mvel = SCHEMA.getField("mvel"); + + public MvelExpression() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public MvelExpression(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static MvelExpression.Fields fields() { + return _fields; + } + + public static MvelExpression.ProjectionMask createMask() { + return new MvelExpression.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for mvel + * + * @see MvelExpression.Fields#mvel + */ + public boolean hasMvel() { + if (_mvelField!= null) { + return true; + } + return super._map.containsKey("mvel"); + } + + /** + * Remover for mvel + * + * @see MvelExpression.Fields#mvel + */ + public void removeMvel() { + super._map.remove("mvel"); + } + + /** + * Getter for mvel + * + * @see MvelExpression.Fields#mvel + */ + public String getMvel(GetMode mode) { + switch (mode) { + case STRICT: + return getMvel(); + case DEFAULT: + case NULL: + if (_mvelField!= null) { + return _mvelField; + } else { + Object __rawValue = super._map.get("mvel"); + _mvelField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _mvelField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for mvel + * + * @return + * Required field. Could be null for partial record. + * @see MvelExpression.Fields#mvel + */ + @Nonnull + public String getMvel() { + if (_mvelField!= null) { + return _mvelField; + } else { + Object __rawValue = super._map.get("mvel"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("mvel"); + } + _mvelField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _mvelField; + } + } + + /** + * Setter for mvel + * + * @see MvelExpression.Fields#mvel + */ + public MvelExpression setMvel(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setMvel(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field mvel of com.linkedin.feathr.compute.MvelExpression"); + } else { + CheckedUtil.putWithoutChecking(super._map, "mvel", value); + _mvelField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeMvel(); + } else { + CheckedUtil.putWithoutChecking(super._map, "mvel", value); + _mvelField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "mvel", value); + _mvelField = value; + } + break; + } + return this; + } + + /** + * Setter for mvel + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see MvelExpression.Fields#mvel + */ + public MvelExpression setMvel( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field mvel of com.linkedin.feathr.compute.MvelExpression to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "mvel", value); + _mvelField = value; + } + return this; + } + + @Override + public MvelExpression clone() + throws CloneNotSupportedException + { + MvelExpression __clone = ((MvelExpression) super.clone()); + __clone.__changeListener = new MvelExpression.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public MvelExpression copy() + throws CloneNotSupportedException + { + MvelExpression __copy = ((MvelExpression) super.copy()); + __copy._mvelField = null; + __copy.__changeListener = new MvelExpression.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final MvelExpression __objectRef; + + private ChangeListener(MvelExpression reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "mvel": + __objectRef._mvelField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The MVEL expression. + * + */ + public PathSpec mvel() { + return new PathSpec(getPathComponents(), "mvel"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(2); + } + + /** + * The MVEL expression. + * + */ + public MvelExpression.ProjectionMask withMvel() { + getDataMap().put("mvel", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeId.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeId.java new file mode 100644 index 000000000..0b78cdadb --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeId.java @@ -0,0 +1,30 @@ + +package com.linkedin.feathr.compute; + +import javax.annotation.Generated; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.TyperefDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.TyperefInfo; + + +/** + * A type ref to int node id + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\NodeId.pdl.") +public class NodeId + extends TyperefInfo +{ + + private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**A type ref to int node id*/typeref NodeId=int", SchemaFormatType.PDL)); + + public NodeId() { + super(SCHEMA); + } + + public static TyperefDataSchema dataSchema() { + return SCHEMA; + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReference.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReference.java new file mode 100644 index 000000000..7d0eedf6e --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReference.java @@ -0,0 +1,488 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the + * original node array. + * For example, consider:- + * anchorA: { + * key: [viewerId, vieweeId] + * feature: featureA + * } + * Let us say featureA is evaluated in node 1. + * derivation: { + * key: [vieweeId, viewerId] + * args1: {key: [vieweeId, viewerId], feature: featureA} + * definition: args1*2 + * } + * Now, the node reference (to represent args1) would be: + * nodeId: 1 + * keyReference: [1,0] - // Indicates the ordering of the key indices. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\NodeReference.pdl.") +public class NodeReference + extends RecordTemplate +{ + + private final static NodeReference.Fields _fields = new NodeReference.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}", SchemaFormatType.PDL)); + private Integer _idField = null; + private KeyReferenceArray _keyReferenceField = null; + private NodeReference.ChangeListener __changeListener = new NodeReference.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); + private final static RecordDataSchema.Field FIELD_KeyReference = SCHEMA.getField("keyReference"); + + public NodeReference() { + super(new DataMap(3, 0.75F), SCHEMA, 2); + addChangeListener(__changeListener); + } + + public NodeReference(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static NodeReference.Fields fields() { + return _fields; + } + + public static NodeReference.ProjectionMask createMask() { + return new NodeReference.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for id + * + * @see NodeReference.Fields#id + */ + public boolean hasId() { + if (_idField!= null) { + return true; + } + return super._map.containsKey("id"); + } + + /** + * Remover for id + * + * @see NodeReference.Fields#id + */ + public void removeId() { + super._map.remove("id"); + } + + /** + * Getter for id + * + * @see NodeReference.Fields#id + */ + public Integer getId(GetMode mode) { + switch (mode) { + case STRICT: + return getId(); + case DEFAULT: + case NULL: + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for id + * + * @return + * Required field. Could be null for partial record. + * @see NodeReference.Fields#id + */ + @Nonnull + public Integer getId() { + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("id"); + } + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + + /** + * Setter for id + * + * @see NodeReference.Fields#id + */ + public NodeReference setId(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setId(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.NodeReference"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeId(); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + } + return this; + } + + /** + * Setter for id + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see NodeReference.Fields#id + */ + public NodeReference setId( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.NodeReference to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + return this; + } + + /** + * Setter for id + * + * @see NodeReference.Fields#id + */ + public NodeReference setId(int value) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + return this; + } + + /** + * Existence checker for keyReference + * + * @see NodeReference.Fields#keyReference + */ + public boolean hasKeyReference() { + if (_keyReferenceField!= null) { + return true; + } + return super._map.containsKey("keyReference"); + } + + /** + * Remover for keyReference + * + * @see NodeReference.Fields#keyReference + */ + public void removeKeyReference() { + super._map.remove("keyReference"); + } + + /** + * Getter for keyReference + * + * @see NodeReference.Fields#keyReference + */ + public KeyReferenceArray getKeyReference(GetMode mode) { + switch (mode) { + case STRICT: + return getKeyReference(); + case DEFAULT: + case NULL: + if (_keyReferenceField!= null) { + return _keyReferenceField; + } else { + Object __rawValue = super._map.get("keyReference"); + _keyReferenceField = ((__rawValue == null)?null:new KeyReferenceArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _keyReferenceField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for keyReference + * + * @return + * Required field. Could be null for partial record. + * @see NodeReference.Fields#keyReference + */ + @Nonnull + public KeyReferenceArray getKeyReference() { + if (_keyReferenceField!= null) { + return _keyReferenceField; + } else { + Object __rawValue = super._map.get("keyReference"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("keyReference"); + } + _keyReferenceField = ((__rawValue == null)?null:new KeyReferenceArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _keyReferenceField; + } + } + + /** + * Setter for keyReference + * + * @see NodeReference.Fields#keyReference + */ + public NodeReference setKeyReference(KeyReferenceArray value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setKeyReference(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field keyReference of com.linkedin.feathr.compute.NodeReference"); + } else { + CheckedUtil.putWithoutChecking(super._map, "keyReference", value.data()); + _keyReferenceField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeKeyReference(); + } else { + CheckedUtil.putWithoutChecking(super._map, "keyReference", value.data()); + _keyReferenceField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "keyReference", value.data()); + _keyReferenceField = value; + } + break; + } + return this; + } + + /** + * Setter for keyReference + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see NodeReference.Fields#keyReference + */ + public NodeReference setKeyReference( + @Nonnull + KeyReferenceArray value) { + if (value == null) { + throw new NullPointerException("Cannot set field keyReference of com.linkedin.feathr.compute.NodeReference to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "keyReference", value.data()); + _keyReferenceField = value; + } + return this; + } + + @Override + public NodeReference clone() + throws CloneNotSupportedException + { + NodeReference __clone = ((NodeReference) super.clone()); + __clone.__changeListener = new NodeReference.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public NodeReference copy() + throws CloneNotSupportedException + { + NodeReference __copy = ((NodeReference) super.copy()); + __copy._keyReferenceField = null; + __copy._idField = null; + __copy.__changeListener = new NodeReference.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final NodeReference __objectRef; + + private ChangeListener(NodeReference reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "keyReference": + __objectRef._keyReferenceField = null; + break; + case "id": + __objectRef._idField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * node id of the referring node. + * + */ + public PathSpec id() { + return new PathSpec(getPathComponents(), "id"); + } + + /** + * The key references in the keys of the referring node. + * + */ + public com.linkedin.feathr.compute.KeyReferenceArray.Fields keyReference() { + return new com.linkedin.feathr.compute.KeyReferenceArray.Fields(getPathComponents(), "keyReference"); + } + + /** + * The key references in the keys of the referring node. + * + */ + public PathSpec keyReference(Integer start, Integer count) { + PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "keyReference"); + if (start!= null) { + arrayPathSpec.setAttribute("start", start); + } + if (count!= null) { + arrayPathSpec.setAttribute("count", count); + } + return arrayPathSpec; + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.KeyReferenceArray.ProjectionMask _keyReferenceMask; + + ProjectionMask() { + super(3); + } + + /** + * node id of the referring node. + * + */ + public NodeReference.ProjectionMask withId() { + getDataMap().put("id", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * The key references in the keys of the referring node. + * + */ + public NodeReference.ProjectionMask withKeyReference(Function nestedMask) { + _keyReferenceMask = nestedMask.apply(((_keyReferenceMask == null)?KeyReferenceArray.createMask():_keyReferenceMask)); + getDataMap().put("keyReference", _keyReferenceMask.getDataMap()); + return this; + } + + /** + * The key references in the keys of the referring node. + * + */ + public NodeReference.ProjectionMask withKeyReference() { + _keyReferenceMask = null; + getDataMap().put("keyReference", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * The key references in the keys of the referring node. + * + */ + public NodeReference.ProjectionMask withKeyReference(Function nestedMask, Integer start, Integer count) { + _keyReferenceMask = nestedMask.apply(((_keyReferenceMask == null)?KeyReferenceArray.createMask():_keyReferenceMask)); + getDataMap().put("keyReference", _keyReferenceMask.getDataMap()); + if (start!= null) { + getDataMap().getDataMap("keyReference").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("keyReference").put("$count", count); + } + return this; + } + + /** + * The key references in the keys of the referring node. + * + */ + public NodeReference.ProjectionMask withKeyReference(Integer start, Integer count) { + _keyReferenceMask = null; + getDataMap().put("keyReference", new DataMap(3)); + if (start!= null) { + getDataMap().getDataMap("keyReference").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("keyReference").put("$count", count); + } + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReferenceArray.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReferenceArray.java new file mode 100644 index 000000000..94b5d095a --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReferenceArray.java @@ -0,0 +1,118 @@ + +package com.linkedin.feathr.compute; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.schema.ArrayDataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.TemplateOutputCastException; +import com.linkedin.data.template.WrappingArrayTemplate; + +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Transformation.pdl.") +public class NodeReferenceArray + extends WrappingArrayTemplate +{ + + private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[{namespace com.linkedin.feathr.compute/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}}]", SchemaFormatType.PDL)); + + public NodeReferenceArray() { + this(new DataList()); + } + + public NodeReferenceArray(int initialCapacity) { + this(new DataList(initialCapacity)); + } + + public NodeReferenceArray(Collection c) { + this(new DataList(c.size())); + addAll(c); + } + + public NodeReferenceArray(DataList data) { + super(data, SCHEMA, NodeReference.class); + } + + public NodeReferenceArray(NodeReference first, NodeReference... rest) { + this(new DataList((rest.length + 1))); + add(first); + addAll(Arrays.asList(rest)); + } + + public static ArrayDataSchema dataSchema() { + return SCHEMA; + } + + public static NodeReferenceArray.ProjectionMask createMask() { + return new NodeReferenceArray.ProjectionMask(); + } + + @Override + public NodeReferenceArray clone() + throws CloneNotSupportedException + { + NodeReferenceArray __clone = ((NodeReferenceArray) super.clone()); + return __clone; + } + + @Override + public NodeReferenceArray copy() + throws CloneNotSupportedException + { + NodeReferenceArray __copy = ((NodeReferenceArray) super.copy()); + return __copy; + } + + @Override + protected NodeReference coerceOutput(Object object) + throws TemplateOutputCastException + { + assert(object != null); + return ((object == null)?null:new NodeReference(DataTemplateUtil.castOrThrow(object, DataMap.class))); + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.compute.NodeReference.Fields items() { + return new com.linkedin.feathr.compute.NodeReference.Fields(getPathComponents(), PathSpec.WILDCARD); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.NodeReference.ProjectionMask _itemsMask; + + ProjectionMask() { + super(4); + } + + public NodeReferenceArray.ProjectionMask withItems(Function nestedMask) { + _itemsMask = nestedMask.apply(((_itemsMask == null)?NodeReference.createMask():_itemsMask)); + getDataMap().put("$*", _itemsMask.getDataMap()); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OfflineKeyFunction.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OfflineKeyFunction.java new file mode 100644 index 000000000..9a62a82ed --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OfflineKeyFunction.java @@ -0,0 +1,477 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.DataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.UnionDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.UnionTemplate; + + +/** + * Represents a feature's key that is extracted from each row of an offline data source and is used to join with observation data to form a training dataset. This class is expected to be included so the definitions of enclosed fields can be reused. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\OfflineKeyFunction.pdl.") +public class OfflineKeyFunction + extends RecordTemplate +{ + + private final static OfflineKeyFunction.Fields _fields = new OfflineKeyFunction.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Represents a feature's key that is extracted from each row of an offline data source and is used to join with observation data to form a training dataset. This class is expected to be included so the definitions of enclosed fields can be reused.*/record OfflineKeyFunction{/**Key function specifies how to extract the feature's key from each row of the offline data source. For example, an offline dataset has x field, a key function being defined as getIdFromUrn(x) means the feature key is a numeric member id, which can later be used to join with observation data that also has numeric member id column. A feature's key can have one key part or multiple key parts (compound key). This field should be required, keeping it optional for fulfilling backward compatiblity requirement during schema evolution.*/keyFunction:optional union[/**An expression in MVEL language. For more information please refer to go/framemvel.*/record MvelExpression{/**The MVEL expression.*/mvel:string}/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}/**User defined function that can be used in feature extraction or derivation.*/record UserDefinedFunction{/**Reference to the class that implements the user defined function.*/clazz:string/**Some UserDefinedFunction requires additional custom parameters. This field defines the custom parameters of the user defined function, represented as a map of string to json blob. The key is the parameter name, and the value is the parameter value represented as a json blob. For example, the parameters may look like: { param1 : [\"waterlooCompany_terms_hashed\", \"waterlooCompany_values\"], param2 : \"com.linkedin.quasar.encoding.SomeEncodingClass\u00e2\u20ac\ufffd } feathr will be responsible of parsing the parameters map into a CustomParameters class defined by application: public class CustomParameters { List param1; String param2; } CustomParameters will be used in the constructor of the UserDefinedFunction.*/parameters:map[string,string]={}}]}", SchemaFormatType.PDL)); + private OfflineKeyFunction.KeyFunction _keyFunctionField = null; + private OfflineKeyFunction.ChangeListener __changeListener = new OfflineKeyFunction.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_KeyFunction = SCHEMA.getField("keyFunction"); + + public OfflineKeyFunction() { + super(new DataMap(2, 0.75F), SCHEMA, 2); + addChangeListener(__changeListener); + } + + public OfflineKeyFunction(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static OfflineKeyFunction.Fields fields() { + return _fields; + } + + public static OfflineKeyFunction.ProjectionMask createMask() { + return new OfflineKeyFunction.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for keyFunction + * + * @see OfflineKeyFunction.Fields#keyFunction + */ + public boolean hasKeyFunction() { + if (_keyFunctionField!= null) { + return true; + } + return super._map.containsKey("keyFunction"); + } + + /** + * Remover for keyFunction + * + * @see OfflineKeyFunction.Fields#keyFunction + */ + public void removeKeyFunction() { + super._map.remove("keyFunction"); + } + + /** + * Getter for keyFunction + * + * @see OfflineKeyFunction.Fields#keyFunction + */ + public OfflineKeyFunction.KeyFunction getKeyFunction(GetMode mode) { + return getKeyFunction(); + } + + /** + * Getter for keyFunction + * + * @return + * Optional field. Always check for null. + * @see OfflineKeyFunction.Fields#keyFunction + */ + @Nullable + public OfflineKeyFunction.KeyFunction getKeyFunction() { + if (_keyFunctionField!= null) { + return _keyFunctionField; + } else { + Object __rawValue = super._map.get("keyFunction"); + _keyFunctionField = ((__rawValue == null)?null:new OfflineKeyFunction.KeyFunction(__rawValue)); + return _keyFunctionField; + } + } + + /** + * Setter for keyFunction + * + * @see OfflineKeyFunction.Fields#keyFunction + */ + public OfflineKeyFunction setKeyFunction(OfflineKeyFunction.KeyFunction value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setKeyFunction(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeKeyFunction(); + } else { + CheckedUtil.putWithoutChecking(super._map, "keyFunction", value.data()); + _keyFunctionField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "keyFunction", value.data()); + _keyFunctionField = value; + } + break; + } + return this; + } + + /** + * Setter for keyFunction + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see OfflineKeyFunction.Fields#keyFunction + */ + public OfflineKeyFunction setKeyFunction( + @Nonnull + OfflineKeyFunction.KeyFunction value) { + if (value == null) { + throw new NullPointerException("Cannot set field keyFunction of com.linkedin.feathr.compute.OfflineKeyFunction to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "keyFunction", value.data()); + _keyFunctionField = value; + } + return this; + } + + @Override + public OfflineKeyFunction clone() + throws CloneNotSupportedException + { + OfflineKeyFunction __clone = ((OfflineKeyFunction) super.clone()); + __clone.__changeListener = new OfflineKeyFunction.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public OfflineKeyFunction copy() + throws CloneNotSupportedException + { + OfflineKeyFunction __copy = ((OfflineKeyFunction) super.copy()); + __copy._keyFunctionField = null; + __copy.__changeListener = new OfflineKeyFunction.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final OfflineKeyFunction __objectRef; + + private ChangeListener(OfflineKeyFunction reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "keyFunction": + __objectRef._keyFunctionField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Key function specifies how to extract the feature's key from each row of the offline data source. For example, an offline dataset has x field, a key function being defined as getIdFromUrn(x) means the feature key is a numeric member id, which can later be used to join with observation data that also has numeric member id column. A feature's key can have one key part or multiple key parts (compound key). This field should be required, keeping it optional for fulfilling backward compatiblity requirement during schema evolution. + * + */ + public com.linkedin.feathr.compute.OfflineKeyFunction.KeyFunction.Fields keyFunction() { + return new com.linkedin.feathr.compute.OfflineKeyFunction.KeyFunction.Fields(getPathComponents(), "keyFunction"); + } + + } + + @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\OfflineKeyFunction.pdl.") + public static class KeyFunction + extends UnionTemplate + { + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**An expression in MVEL language. For more information please refer to go/framemvel.*/record MvelExpression{/**The MVEL expression.*/mvel:string}}{namespace com.linkedin.feathr.compute/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}}{namespace com.linkedin.feathr.compute/**User defined function that can be used in feature extraction or derivation.*/record UserDefinedFunction{/**Reference to the class that implements the user defined function.*/clazz:string/**Some UserDefinedFunction requires additional custom parameters. This field defines the custom parameters of the user defined function, represented as a map of string to json blob. The key is the parameter name, and the value is the parameter value represented as a json blob. For example, the parameters may look like: { param1 : [\"waterlooCompany_terms_hashed\", \"waterlooCompany_values\"], param2 : \"com.linkedin.quasar.encoding.SomeEncodingClass\u00e2\u20ac\ufffd } feathr will be responsible of parsing the parameters map into a CustomParameters class defined by application: public class CustomParameters { List param1; String param2; } CustomParameters will be used in the constructor of the UserDefinedFunction.*/parameters:map[string,string]={}}}]", SchemaFormatType.PDL)); + private com.linkedin.feathr.compute.MvelExpression _mvelExpressionMember = null; + private com.linkedin.feathr.compute.SqlExpression _sqlExpressionMember = null; + private com.linkedin.feathr.compute.UserDefinedFunction _userDefinedFunctionMember = null; + private OfflineKeyFunction.KeyFunction.ChangeListener __changeListener = new OfflineKeyFunction.KeyFunction.ChangeListener(this); + private final static DataSchema MEMBER_MvelExpression = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.MvelExpression"); + private final static DataSchema MEMBER_SqlExpression = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.SqlExpression"); + private final static DataSchema MEMBER_UserDefinedFunction = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.UserDefinedFunction"); + + public KeyFunction() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public KeyFunction(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static OfflineKeyFunction.KeyFunction create(com.linkedin.feathr.compute.MvelExpression value) { + OfflineKeyFunction.KeyFunction newUnion = new OfflineKeyFunction.KeyFunction(); + newUnion.setMvelExpression(value); + return newUnion; + } + + public boolean isMvelExpression() { + return memberIs("com.linkedin.feathr.compute.MvelExpression"); + } + + public com.linkedin.feathr.compute.MvelExpression getMvelExpression() { + checkNotNull(); + if (_mvelExpressionMember!= null) { + return _mvelExpressionMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.MvelExpression"); + _mvelExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.MvelExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _mvelExpressionMember; + } + + public void setMvelExpression(com.linkedin.feathr.compute.MvelExpression value) { + checkNotNull(); + super._map.clear(); + _mvelExpressionMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.MvelExpression", value.data()); + } + + public static OfflineKeyFunction.KeyFunction create(com.linkedin.feathr.compute.SqlExpression value) { + OfflineKeyFunction.KeyFunction newUnion = new OfflineKeyFunction.KeyFunction(); + newUnion.setSqlExpression(value); + return newUnion; + } + + public boolean isSqlExpression() { + return memberIs("com.linkedin.feathr.compute.SqlExpression"); + } + + public com.linkedin.feathr.compute.SqlExpression getSqlExpression() { + checkNotNull(); + if (_sqlExpressionMember!= null) { + return _sqlExpressionMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.SqlExpression"); + _sqlExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.SqlExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _sqlExpressionMember; + } + + public void setSqlExpression(com.linkedin.feathr.compute.SqlExpression value) { + checkNotNull(); + super._map.clear(); + _sqlExpressionMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.SqlExpression", value.data()); + } + + public static OfflineKeyFunction.KeyFunction create(com.linkedin.feathr.compute.UserDefinedFunction value) { + OfflineKeyFunction.KeyFunction newUnion = new OfflineKeyFunction.KeyFunction(); + newUnion.setUserDefinedFunction(value); + return newUnion; + } + + public boolean isUserDefinedFunction() { + return memberIs("com.linkedin.feathr.compute.UserDefinedFunction"); + } + + public com.linkedin.feathr.compute.UserDefinedFunction getUserDefinedFunction() { + checkNotNull(); + if (_userDefinedFunctionMember!= null) { + return _userDefinedFunctionMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.UserDefinedFunction"); + _userDefinedFunctionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.UserDefinedFunction(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _userDefinedFunctionMember; + } + + public void setUserDefinedFunction(com.linkedin.feathr.compute.UserDefinedFunction value) { + checkNotNull(); + super._map.clear(); + _userDefinedFunctionMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.UserDefinedFunction", value.data()); + } + + public static OfflineKeyFunction.KeyFunction.ProjectionMask createMask() { + return new OfflineKeyFunction.KeyFunction.ProjectionMask(); + } + + @Override + public OfflineKeyFunction.KeyFunction clone() + throws CloneNotSupportedException + { + OfflineKeyFunction.KeyFunction __clone = ((OfflineKeyFunction.KeyFunction) super.clone()); + __clone.__changeListener = new OfflineKeyFunction.KeyFunction.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public OfflineKeyFunction.KeyFunction copy() + throws CloneNotSupportedException + { + OfflineKeyFunction.KeyFunction __copy = ((OfflineKeyFunction.KeyFunction) super.copy()); + __copy._sqlExpressionMember = null; + __copy._userDefinedFunctionMember = null; + __copy._mvelExpressionMember = null; + __copy.__changeListener = new OfflineKeyFunction.KeyFunction.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final OfflineKeyFunction.KeyFunction __objectRef; + + private ChangeListener(OfflineKeyFunction.KeyFunction reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "com.linkedin.feathr.compute.SqlExpression": + __objectRef._sqlExpressionMember = null; + break; + case "com.linkedin.feathr.compute.UserDefinedFunction": + __objectRef._userDefinedFunctionMember = null; + break; + case "com.linkedin.feathr.compute.MvelExpression": + __objectRef._mvelExpressionMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.compute.MvelExpression.Fields MvelExpression() { + return new com.linkedin.feathr.compute.MvelExpression.Fields(getPathComponents(), "com.linkedin.feathr.compute.MvelExpression"); + } + + public com.linkedin.feathr.compute.SqlExpression.Fields SqlExpression() { + return new com.linkedin.feathr.compute.SqlExpression.Fields(getPathComponents(), "com.linkedin.feathr.compute.SqlExpression"); + } + + public com.linkedin.feathr.compute.UserDefinedFunction.Fields UserDefinedFunction() { + return new com.linkedin.feathr.compute.UserDefinedFunction.Fields(getPathComponents(), "com.linkedin.feathr.compute.UserDefinedFunction"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.MvelExpression.ProjectionMask _MvelExpressionMask; + private com.linkedin.feathr.compute.SqlExpression.ProjectionMask _SqlExpressionMask; + private com.linkedin.feathr.compute.UserDefinedFunction.ProjectionMask _UserDefinedFunctionMask; + + ProjectionMask() { + super(4); + } + + public OfflineKeyFunction.KeyFunction.ProjectionMask withMvelExpression(Function nestedMask) { + _MvelExpressionMask = nestedMask.apply(((_MvelExpressionMask == null)?com.linkedin.feathr.compute.MvelExpression.createMask():_MvelExpressionMask)); + getDataMap().put("com.linkedin.feathr.compute.MvelExpression", _MvelExpressionMask.getDataMap()); + return this; + } + + public OfflineKeyFunction.KeyFunction.ProjectionMask withSqlExpression(Function nestedMask) { + _SqlExpressionMask = nestedMask.apply(((_SqlExpressionMask == null)?com.linkedin.feathr.compute.SqlExpression.createMask():_SqlExpressionMask)); + getDataMap().put("com.linkedin.feathr.compute.SqlExpression", _SqlExpressionMask.getDataMap()); + return this; + } + + public OfflineKeyFunction.KeyFunction.ProjectionMask withUserDefinedFunction(Function nestedMask) { + _UserDefinedFunctionMask = nestedMask.apply(((_UserDefinedFunctionMask == null)?com.linkedin.feathr.compute.UserDefinedFunction.createMask():_UserDefinedFunctionMask)); + getDataMap().put("com.linkedin.feathr.compute.UserDefinedFunction", _UserDefinedFunctionMask.getDataMap()); + return this; + } + + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.OfflineKeyFunction.KeyFunction.ProjectionMask _keyFunctionMask; + + ProjectionMask() { + super(2); + } + + /** + * Key function specifies how to extract the feature's key from each row of the offline data source. For example, an offline dataset has x field, a key function being defined as getIdFromUrn(x) means the feature key is a numeric member id, which can later be used to join with observation data that also has numeric member id column. A feature's key can have one key part or multiple key parts (compound key). This field should be required, keeping it optional for fulfilling backward compatiblity requirement during schema evolution. + * + */ + public OfflineKeyFunction.ProjectionMask withKeyFunction(Function nestedMask) { + _keyFunctionMask = nestedMask.apply(((_keyFunctionMask == null)?OfflineKeyFunction.KeyFunction.createMask():_keyFunctionMask)); + getDataMap().put("keyFunction", _keyFunctionMask.getDataMap()); + return this; + } + + /** + * Key function specifies how to extract the feature's key from each row of the offline data source. For example, an offline dataset has x field, a key function being defined as getIdFromUrn(x) means the feature key is a numeric member id, which can later be used to join with observation data that also has numeric member id column. A feature's key can have one key part or multiple key parts (compound key). This field should be required, keeping it optional for fulfilling backward compatiblity requirement during schema evolution. + * + */ + public OfflineKeyFunction.ProjectionMask withKeyFunction() { + _keyFunctionMask = null; + getDataMap().put("keyFunction", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OperatorId.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OperatorId.java new file mode 100644 index 000000000..1a1364317 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OperatorId.java @@ -0,0 +1,30 @@ + +package com.linkedin.feathr.compute; + +import javax.annotation.Generated; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.TyperefDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.TyperefInfo; + + +/** + * operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\OperatorId.pdl.") +public class OperatorId + extends TyperefInfo +{ + + private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string", SchemaFormatType.PDL)); + + public OperatorId() { + super(SCHEMA); + } + + public static TyperefDataSchema dataSchema() { + return SCHEMA; + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SlidingWindowFeature.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SlidingWindowFeature.java new file mode 100644 index 000000000..f6ba7dc67 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SlidingWindowFeature.java @@ -0,0 +1,1574 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.DataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.UnionDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.UnionTemplate; + + +/** + * Sliding window aggregation produces feature data by aggregating a collection of data within a given time interval into an aggregate value. It ensures point-in-time correctness, when joining with label data, feathr looks back the configurable time window from each entry's timestamp and compute the aggregagate value. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SlidingWindowFeature.pdl.") +public class SlidingWindowFeature + extends RecordTemplate +{ + + private final static SlidingWindowFeature.Fields _fields = new SlidingWindowFeature.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Sliding window aggregation produces feature data by aggregating a collection of data within a given time interval into an aggregate value. It ensures point-in-time correctness, when joining with label data, feathr looks back the configurable time window from each entry's timestamp and compute the aggregagate value.*/record SlidingWindowFeature{/**The target column to perform aggregation against.*/targetColumn:union[/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}]/**Represents supported types of aggregation.*/aggregationType:enum AggregationType{/** Sum. */SUM/** Count. */COUNT/** Max. */MAX/** Min. */MIN/** Average. */AVG/** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Max pooling is done by applying a max filter to (usually) non-overlapping subregions of the initial representation. */MAX_POOLING/** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Min pooling is done by applying a min filter to (usually) non-overlapping subregions of the initial representation. */MIN_POOLING/** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Average pooling is done by applying a average filter to (usually) non-overlapping subregions of the initial representation. */AVG_POOLING/** Latest */LATEST}/**Represents the time window to look back from label data's timestamp.*/window:/**Represents a time window used in sliding window algorithms.*/record Window{/**Represents the duration of the window.*/size:int/**Represents a unit of time.*/unit:enum Unit{/** A day. */DAY/** An hour. */HOUR/** A minute. */MINUTE/** A second. */SECOND}}/**Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details.*/lateralViews:array[/**Lateral view is used in conjunction with table generating functions (eg. the most commonly used explode()), which typically generates zero or more output rows for each input row. A lateral view first applies the table generating function to each row of base table, and then joins resulting output rows to the input rows to form a virtual table with the supplied table alias. For more details and examples, refer to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+LateralView.*/record LateralView{/**A table-generating function transforms a single input row to multiple output rows. For example, explode(array('A','B','C') will produce 3 one-column rows, which are row1: 'A'; row2: 'B'; row3: 'C'.*/tableGeneratingFunction:union[SqlExpression]/**Represents the alias for referencing the generated virtual table. It will be used in subsequent statements (eg. filter, groupBy) in the sliding window feature definition.*/virtualTableAlias:string}]=[]/**Represents the filter statement before the aggregation.*/filter:optional union[SqlExpression]/**Represents the target to be grouped by before aggregation. If groupBy is not set, the aggregation will be performed over the entire dataset.*/groupBy:optional union[SqlExpression]/**Represents the max number of groups (with aggregation results) to return.*/limit:optional int}", SchemaFormatType.PDL)); + private SlidingWindowFeature.TargetColumn _targetColumnField = null; + private AggregationType _aggregationTypeField = null; + private Window _windowField = null; + private LateralViewArray _lateralViewsField = null; + private SlidingWindowFeature.Filter _filterField = null; + private SlidingWindowFeature.GroupBy _groupByField = null; + private Integer _limitField = null; + private SlidingWindowFeature.ChangeListener __changeListener = new SlidingWindowFeature.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_TargetColumn = SCHEMA.getField("targetColumn"); + private final static RecordDataSchema.Field FIELD_AggregationType = SCHEMA.getField("aggregationType"); + private final static RecordDataSchema.Field FIELD_Window = SCHEMA.getField("window"); + private final static RecordDataSchema.Field FIELD_LateralViews = SCHEMA.getField("lateralViews"); + private final static LateralViewArray DEFAULT_LateralViews; + private final static RecordDataSchema.Field FIELD_Filter = SCHEMA.getField("filter"); + private final static RecordDataSchema.Field FIELD_GroupBy = SCHEMA.getField("groupBy"); + private final static RecordDataSchema.Field FIELD_Limit = SCHEMA.getField("limit"); + + static { + DEFAULT_LateralViews = ((FIELD_LateralViews.getDefault() == null)?null:new LateralViewArray(DataTemplateUtil.castOrThrow(FIELD_LateralViews.getDefault(), DataList.class))); + } + + public SlidingWindowFeature() { + super(new DataMap(10, 0.75F), SCHEMA, 7); + addChangeListener(__changeListener); + } + + public SlidingWindowFeature(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static SlidingWindowFeature.Fields fields() { + return _fields; + } + + public static SlidingWindowFeature.ProjectionMask createMask() { + return new SlidingWindowFeature.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for targetColumn + * + * @see SlidingWindowFeature.Fields#targetColumn + */ + public boolean hasTargetColumn() { + if (_targetColumnField!= null) { + return true; + } + return super._map.containsKey("targetColumn"); + } + + /** + * Remover for targetColumn + * + * @see SlidingWindowFeature.Fields#targetColumn + */ + public void removeTargetColumn() { + super._map.remove("targetColumn"); + } + + /** + * Getter for targetColumn + * + * @see SlidingWindowFeature.Fields#targetColumn + */ + public SlidingWindowFeature.TargetColumn getTargetColumn(GetMode mode) { + switch (mode) { + case STRICT: + return getTargetColumn(); + case DEFAULT: + case NULL: + if (_targetColumnField!= null) { + return _targetColumnField; + } else { + Object __rawValue = super._map.get("targetColumn"); + _targetColumnField = ((__rawValue == null)?null:new SlidingWindowFeature.TargetColumn(__rawValue)); + return _targetColumnField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for targetColumn + * + * @return + * Required field. Could be null for partial record. + * @see SlidingWindowFeature.Fields#targetColumn + */ + @Nonnull + public SlidingWindowFeature.TargetColumn getTargetColumn() { + if (_targetColumnField!= null) { + return _targetColumnField; + } else { + Object __rawValue = super._map.get("targetColumn"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("targetColumn"); + } + _targetColumnField = ((__rawValue == null)?null:new SlidingWindowFeature.TargetColumn(__rawValue)); + return _targetColumnField; + } + } + + /** + * Setter for targetColumn + * + * @see SlidingWindowFeature.Fields#targetColumn + */ + public SlidingWindowFeature setTargetColumn(SlidingWindowFeature.TargetColumn value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setTargetColumn(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field targetColumn of com.linkedin.feathr.compute.SlidingWindowFeature"); + } else { + CheckedUtil.putWithoutChecking(super._map, "targetColumn", value.data()); + _targetColumnField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeTargetColumn(); + } else { + CheckedUtil.putWithoutChecking(super._map, "targetColumn", value.data()); + _targetColumnField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "targetColumn", value.data()); + _targetColumnField = value; + } + break; + } + return this; + } + + /** + * Setter for targetColumn + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see SlidingWindowFeature.Fields#targetColumn + */ + public SlidingWindowFeature setTargetColumn( + @Nonnull + SlidingWindowFeature.TargetColumn value) { + if (value == null) { + throw new NullPointerException("Cannot set field targetColumn of com.linkedin.feathr.compute.SlidingWindowFeature to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "targetColumn", value.data()); + _targetColumnField = value; + } + return this; + } + + /** + * Existence checker for aggregationType + * + * @see SlidingWindowFeature.Fields#aggregationType + */ + public boolean hasAggregationType() { + if (_aggregationTypeField!= null) { + return true; + } + return super._map.containsKey("aggregationType"); + } + + /** + * Remover for aggregationType + * + * @see SlidingWindowFeature.Fields#aggregationType + */ + public void removeAggregationType() { + super._map.remove("aggregationType"); + } + + /** + * Getter for aggregationType + * + * @see SlidingWindowFeature.Fields#aggregationType + */ + public AggregationType getAggregationType(GetMode mode) { + switch (mode) { + case STRICT: + return getAggregationType(); + case DEFAULT: + case NULL: + if (_aggregationTypeField!= null) { + return _aggregationTypeField; + } else { + Object __rawValue = super._map.get("aggregationType"); + _aggregationTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, AggregationType.class, AggregationType.$UNKNOWN); + return _aggregationTypeField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for aggregationType + * + * @return + * Required field. Could be null for partial record. + * @see SlidingWindowFeature.Fields#aggregationType + */ + @Nonnull + public AggregationType getAggregationType() { + if (_aggregationTypeField!= null) { + return _aggregationTypeField; + } else { + Object __rawValue = super._map.get("aggregationType"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("aggregationType"); + } + _aggregationTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, AggregationType.class, AggregationType.$UNKNOWN); + return _aggregationTypeField; + } + } + + /** + * Setter for aggregationType + * + * @see SlidingWindowFeature.Fields#aggregationType + */ + public SlidingWindowFeature setAggregationType(AggregationType value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setAggregationType(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field aggregationType of com.linkedin.feathr.compute.SlidingWindowFeature"); + } else { + CheckedUtil.putWithoutChecking(super._map, "aggregationType", value.name()); + _aggregationTypeField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeAggregationType(); + } else { + CheckedUtil.putWithoutChecking(super._map, "aggregationType", value.name()); + _aggregationTypeField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "aggregationType", value.name()); + _aggregationTypeField = value; + } + break; + } + return this; + } + + /** + * Setter for aggregationType + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see SlidingWindowFeature.Fields#aggregationType + */ + public SlidingWindowFeature setAggregationType( + @Nonnull + AggregationType value) { + if (value == null) { + throw new NullPointerException("Cannot set field aggregationType of com.linkedin.feathr.compute.SlidingWindowFeature to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "aggregationType", value.name()); + _aggregationTypeField = value; + } + return this; + } + + /** + * Existence checker for window + * + * @see SlidingWindowFeature.Fields#window + */ + public boolean hasWindow() { + if (_windowField!= null) { + return true; + } + return super._map.containsKey("window"); + } + + /** + * Remover for window + * + * @see SlidingWindowFeature.Fields#window + */ + public void removeWindow() { + super._map.remove("window"); + } + + /** + * Getter for window + * + * @see SlidingWindowFeature.Fields#window + */ + public Window getWindow(GetMode mode) { + switch (mode) { + case STRICT: + return getWindow(); + case DEFAULT: + case NULL: + if (_windowField!= null) { + return _windowField; + } else { + Object __rawValue = super._map.get("window"); + _windowField = ((__rawValue == null)?null:new Window(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _windowField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for window + * + * @return + * Required field. Could be null for partial record. + * @see SlidingWindowFeature.Fields#window + */ + @Nonnull + public Window getWindow() { + if (_windowField!= null) { + return _windowField; + } else { + Object __rawValue = super._map.get("window"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("window"); + } + _windowField = ((__rawValue == null)?null:new Window(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _windowField; + } + } + + /** + * Setter for window + * + * @see SlidingWindowFeature.Fields#window + */ + public SlidingWindowFeature setWindow(Window value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setWindow(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field window of com.linkedin.feathr.compute.SlidingWindowFeature"); + } else { + CheckedUtil.putWithoutChecking(super._map, "window", value.data()); + _windowField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeWindow(); + } else { + CheckedUtil.putWithoutChecking(super._map, "window", value.data()); + _windowField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "window", value.data()); + _windowField = value; + } + break; + } + return this; + } + + /** + * Setter for window + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see SlidingWindowFeature.Fields#window + */ + public SlidingWindowFeature setWindow( + @Nonnull + Window value) { + if (value == null) { + throw new NullPointerException("Cannot set field window of com.linkedin.feathr.compute.SlidingWindowFeature to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "window", value.data()); + _windowField = value; + } + return this; + } + + /** + * Existence checker for lateralViews + * + * @see SlidingWindowFeature.Fields#lateralViews + */ + public boolean hasLateralViews() { + if (_lateralViewsField!= null) { + return true; + } + return super._map.containsKey("lateralViews"); + } + + /** + * Remover for lateralViews + * + * @see SlidingWindowFeature.Fields#lateralViews + */ + public void removeLateralViews() { + super._map.remove("lateralViews"); + } + + /** + * Getter for lateralViews + * + * @see SlidingWindowFeature.Fields#lateralViews + */ + public LateralViewArray getLateralViews(GetMode mode) { + switch (mode) { + case STRICT: + case DEFAULT: + return getLateralViews(); + case NULL: + if (_lateralViewsField!= null) { + return _lateralViewsField; + } else { + Object __rawValue = super._map.get("lateralViews"); + _lateralViewsField = ((__rawValue == null)?null:new LateralViewArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _lateralViewsField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for lateralViews + * + * @return + * Required field. Could be null for partial record. + * @see SlidingWindowFeature.Fields#lateralViews + */ + @Nonnull + public LateralViewArray getLateralViews() { + if (_lateralViewsField!= null) { + return _lateralViewsField; + } else { + Object __rawValue = super._map.get("lateralViews"); + if (__rawValue == null) { + return DEFAULT_LateralViews; + } + _lateralViewsField = ((__rawValue == null)?null:new LateralViewArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _lateralViewsField; + } + } + + /** + * Setter for lateralViews + * + * @see SlidingWindowFeature.Fields#lateralViews + */ + public SlidingWindowFeature setLateralViews(LateralViewArray value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setLateralViews(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field lateralViews of com.linkedin.feathr.compute.SlidingWindowFeature"); + } else { + CheckedUtil.putWithoutChecking(super._map, "lateralViews", value.data()); + _lateralViewsField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeLateralViews(); + } else { + CheckedUtil.putWithoutChecking(super._map, "lateralViews", value.data()); + _lateralViewsField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "lateralViews", value.data()); + _lateralViewsField = value; + } + break; + } + return this; + } + + /** + * Setter for lateralViews + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see SlidingWindowFeature.Fields#lateralViews + */ + public SlidingWindowFeature setLateralViews( + @Nonnull + LateralViewArray value) { + if (value == null) { + throw new NullPointerException("Cannot set field lateralViews of com.linkedin.feathr.compute.SlidingWindowFeature to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "lateralViews", value.data()); + _lateralViewsField = value; + } + return this; + } + + /** + * Existence checker for filter + * + * @see SlidingWindowFeature.Fields#filter + */ + public boolean hasFilter() { + if (_filterField!= null) { + return true; + } + return super._map.containsKey("filter"); + } + + /** + * Remover for filter + * + * @see SlidingWindowFeature.Fields#filter + */ + public void removeFilter() { + super._map.remove("filter"); + } + + /** + * Getter for filter + * + * @see SlidingWindowFeature.Fields#filter + */ + public SlidingWindowFeature.Filter getFilter(GetMode mode) { + return getFilter(); + } + + /** + * Getter for filter + * + * @return + * Optional field. Always check for null. + * @see SlidingWindowFeature.Fields#filter + */ + @Nullable + public SlidingWindowFeature.Filter getFilter() { + if (_filterField!= null) { + return _filterField; + } else { + Object __rawValue = super._map.get("filter"); + _filterField = ((__rawValue == null)?null:new SlidingWindowFeature.Filter(__rawValue)); + return _filterField; + } + } + + /** + * Setter for filter + * + * @see SlidingWindowFeature.Fields#filter + */ + public SlidingWindowFeature setFilter(SlidingWindowFeature.Filter value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFilter(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeFilter(); + } else { + CheckedUtil.putWithoutChecking(super._map, "filter", value.data()); + _filterField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "filter", value.data()); + _filterField = value; + } + break; + } + return this; + } + + /** + * Setter for filter + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see SlidingWindowFeature.Fields#filter + */ + public SlidingWindowFeature setFilter( + @Nonnull + SlidingWindowFeature.Filter value) { + if (value == null) { + throw new NullPointerException("Cannot set field filter of com.linkedin.feathr.compute.SlidingWindowFeature to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "filter", value.data()); + _filterField = value; + } + return this; + } + + /** + * Existence checker for groupBy + * + * @see SlidingWindowFeature.Fields#groupBy + */ + public boolean hasGroupBy() { + if (_groupByField!= null) { + return true; + } + return super._map.containsKey("groupBy"); + } + + /** + * Remover for groupBy + * + * @see SlidingWindowFeature.Fields#groupBy + */ + public void removeGroupBy() { + super._map.remove("groupBy"); + } + + /** + * Getter for groupBy + * + * @see SlidingWindowFeature.Fields#groupBy + */ + public SlidingWindowFeature.GroupBy getGroupBy(GetMode mode) { + return getGroupBy(); + } + + /** + * Getter for groupBy + * + * @return + * Optional field. Always check for null. + * @see SlidingWindowFeature.Fields#groupBy + */ + @Nullable + public SlidingWindowFeature.GroupBy getGroupBy() { + if (_groupByField!= null) { + return _groupByField; + } else { + Object __rawValue = super._map.get("groupBy"); + _groupByField = ((__rawValue == null)?null:new SlidingWindowFeature.GroupBy(__rawValue)); + return _groupByField; + } + } + + /** + * Setter for groupBy + * + * @see SlidingWindowFeature.Fields#groupBy + */ + public SlidingWindowFeature setGroupBy(SlidingWindowFeature.GroupBy value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setGroupBy(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeGroupBy(); + } else { + CheckedUtil.putWithoutChecking(super._map, "groupBy", value.data()); + _groupByField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "groupBy", value.data()); + _groupByField = value; + } + break; + } + return this; + } + + /** + * Setter for groupBy + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see SlidingWindowFeature.Fields#groupBy + */ + public SlidingWindowFeature setGroupBy( + @Nonnull + SlidingWindowFeature.GroupBy value) { + if (value == null) { + throw new NullPointerException("Cannot set field groupBy of com.linkedin.feathr.compute.SlidingWindowFeature to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "groupBy", value.data()); + _groupByField = value; + } + return this; + } + + /** + * Existence checker for limit + * + * @see SlidingWindowFeature.Fields#limit + */ + public boolean hasLimit() { + if (_limitField!= null) { + return true; + } + return super._map.containsKey("limit"); + } + + /** + * Remover for limit + * + * @see SlidingWindowFeature.Fields#limit + */ + public void removeLimit() { + super._map.remove("limit"); + } + + /** + * Getter for limit + * + * @see SlidingWindowFeature.Fields#limit + */ + public Integer getLimit(GetMode mode) { + return getLimit(); + } + + /** + * Getter for limit + * + * @return + * Optional field. Always check for null. + * @see SlidingWindowFeature.Fields#limit + */ + @Nullable + public Integer getLimit() { + if (_limitField!= null) { + return _limitField; + } else { + Object __rawValue = super._map.get("limit"); + _limitField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _limitField; + } + } + + /** + * Setter for limit + * + * @see SlidingWindowFeature.Fields#limit + */ + public SlidingWindowFeature setLimit(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setLimit(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeLimit(); + } else { + CheckedUtil.putWithoutChecking(super._map, "limit", DataTemplateUtil.coerceIntInput(value)); + _limitField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "limit", DataTemplateUtil.coerceIntInput(value)); + _limitField = value; + } + break; + } + return this; + } + + /** + * Setter for limit + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see SlidingWindowFeature.Fields#limit + */ + public SlidingWindowFeature setLimit( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field limit of com.linkedin.feathr.compute.SlidingWindowFeature to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "limit", DataTemplateUtil.coerceIntInput(value)); + _limitField = value; + } + return this; + } + + /** + * Setter for limit + * + * @see SlidingWindowFeature.Fields#limit + */ + public SlidingWindowFeature setLimit(int value) { + CheckedUtil.putWithoutChecking(super._map, "limit", DataTemplateUtil.coerceIntInput(value)); + _limitField = value; + return this; + } + + @Override + public SlidingWindowFeature clone() + throws CloneNotSupportedException + { + SlidingWindowFeature __clone = ((SlidingWindowFeature) super.clone()); + __clone.__changeListener = new SlidingWindowFeature.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public SlidingWindowFeature copy() + throws CloneNotSupportedException + { + SlidingWindowFeature __copy = ((SlidingWindowFeature) super.copy()); + __copy._filterField = null; + __copy._aggregationTypeField = null; + __copy._targetColumnField = null; + __copy._limitField = null; + __copy._windowField = null; + __copy._groupByField = null; + __copy._lateralViewsField = null; + __copy.__changeListener = new SlidingWindowFeature.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final SlidingWindowFeature __objectRef; + + private ChangeListener(SlidingWindowFeature reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "filter": + __objectRef._filterField = null; + break; + case "aggregationType": + __objectRef._aggregationTypeField = null; + break; + case "targetColumn": + __objectRef._targetColumnField = null; + break; + case "limit": + __objectRef._limitField = null; + break; + case "window": + __objectRef._windowField = null; + break; + case "groupBy": + __objectRef._groupByField = null; + break; + case "lateralViews": + __objectRef._lateralViewsField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The target column to perform aggregation against. + * + */ + public com.linkedin.feathr.compute.SlidingWindowFeature.TargetColumn.Fields targetColumn() { + return new com.linkedin.feathr.compute.SlidingWindowFeature.TargetColumn.Fields(getPathComponents(), "targetColumn"); + } + + /** + * Represents supported types of aggregation. + * + */ + public PathSpec aggregationType() { + return new PathSpec(getPathComponents(), "aggregationType"); + } + + /** + * Represents the time window to look back from label data's timestamp. + * + */ + public com.linkedin.feathr.compute.Window.Fields window() { + return new com.linkedin.feathr.compute.Window.Fields(getPathComponents(), "window"); + } + + /** + * Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details. + * + */ + public com.linkedin.feathr.compute.LateralViewArray.Fields lateralViews() { + return new com.linkedin.feathr.compute.LateralViewArray.Fields(getPathComponents(), "lateralViews"); + } + + /** + * Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details. + * + */ + public PathSpec lateralViews(Integer start, Integer count) { + PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "lateralViews"); + if (start!= null) { + arrayPathSpec.setAttribute("start", start); + } + if (count!= null) { + arrayPathSpec.setAttribute("count", count); + } + return arrayPathSpec; + } + + /** + * Represents the filter statement before the aggregation. + * + */ + public com.linkedin.feathr.compute.SlidingWindowFeature.Filter.Fields filter() { + return new com.linkedin.feathr.compute.SlidingWindowFeature.Filter.Fields(getPathComponents(), "filter"); + } + + /** + * Represents the target to be grouped by before aggregation. If groupBy is not set, the aggregation will be performed over the entire dataset. + * + */ + public com.linkedin.feathr.compute.SlidingWindowFeature.GroupBy.Fields groupBy() { + return new com.linkedin.feathr.compute.SlidingWindowFeature.GroupBy.Fields(getPathComponents(), "groupBy"); + } + + /** + * Represents the max number of groups (with aggregation results) to return. + * + */ + public PathSpec limit() { + return new PathSpec(getPathComponents(), "limit"); + } + + } + + @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SlidingWindowFeature.pdl.") + public static class Filter + extends UnionTemplate + { + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}}]", SchemaFormatType.PDL)); + private com.linkedin.feathr.compute.SqlExpression _sqlExpressionMember = null; + private SlidingWindowFeature.Filter.ChangeListener __changeListener = new SlidingWindowFeature.Filter.ChangeListener(this); + private final static DataSchema MEMBER_SqlExpression = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.SqlExpression"); + + public Filter() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public Filter(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static SlidingWindowFeature.Filter create(com.linkedin.feathr.compute.SqlExpression value) { + SlidingWindowFeature.Filter newUnion = new SlidingWindowFeature.Filter(); + newUnion.setSqlExpression(value); + return newUnion; + } + + public boolean isSqlExpression() { + return memberIs("com.linkedin.feathr.compute.SqlExpression"); + } + + public com.linkedin.feathr.compute.SqlExpression getSqlExpression() { + checkNotNull(); + if (_sqlExpressionMember!= null) { + return _sqlExpressionMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.SqlExpression"); + _sqlExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.SqlExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _sqlExpressionMember; + } + + public void setSqlExpression(com.linkedin.feathr.compute.SqlExpression value) { + checkNotNull(); + super._map.clear(); + _sqlExpressionMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.SqlExpression", value.data()); + } + + public static SlidingWindowFeature.Filter.ProjectionMask createMask() { + return new SlidingWindowFeature.Filter.ProjectionMask(); + } + + @Override + public SlidingWindowFeature.Filter clone() + throws CloneNotSupportedException + { + SlidingWindowFeature.Filter __clone = ((SlidingWindowFeature.Filter) super.clone()); + __clone.__changeListener = new SlidingWindowFeature.Filter.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public SlidingWindowFeature.Filter copy() + throws CloneNotSupportedException + { + SlidingWindowFeature.Filter __copy = ((SlidingWindowFeature.Filter) super.copy()); + __copy._sqlExpressionMember = null; + __copy.__changeListener = new SlidingWindowFeature.Filter.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final SlidingWindowFeature.Filter __objectRef; + + private ChangeListener(SlidingWindowFeature.Filter reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "com.linkedin.feathr.compute.SqlExpression": + __objectRef._sqlExpressionMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.compute.SqlExpression.Fields SqlExpression() { + return new com.linkedin.feathr.compute.SqlExpression.Fields(getPathComponents(), "com.linkedin.feathr.compute.SqlExpression"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.SqlExpression.ProjectionMask _SqlExpressionMask; + + ProjectionMask() { + super(2); + } + + public SlidingWindowFeature.Filter.ProjectionMask withSqlExpression(Function nestedMask) { + _SqlExpressionMask = nestedMask.apply(((_SqlExpressionMask == null)?com.linkedin.feathr.compute.SqlExpression.createMask():_SqlExpressionMask)); + getDataMap().put("com.linkedin.feathr.compute.SqlExpression", _SqlExpressionMask.getDataMap()); + return this; + } + + } + + } + + @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SlidingWindowFeature.pdl.") + public static class GroupBy + extends UnionTemplate + { + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}}]", SchemaFormatType.PDL)); + private com.linkedin.feathr.compute.SqlExpression _sqlExpressionMember = null; + private SlidingWindowFeature.GroupBy.ChangeListener __changeListener = new SlidingWindowFeature.GroupBy.ChangeListener(this); + private final static DataSchema MEMBER_SqlExpression = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.SqlExpression"); + + public GroupBy() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public GroupBy(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static SlidingWindowFeature.GroupBy create(com.linkedin.feathr.compute.SqlExpression value) { + SlidingWindowFeature.GroupBy newUnion = new SlidingWindowFeature.GroupBy(); + newUnion.setSqlExpression(value); + return newUnion; + } + + public boolean isSqlExpression() { + return memberIs("com.linkedin.feathr.compute.SqlExpression"); + } + + public com.linkedin.feathr.compute.SqlExpression getSqlExpression() { + checkNotNull(); + if (_sqlExpressionMember!= null) { + return _sqlExpressionMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.SqlExpression"); + _sqlExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.SqlExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _sqlExpressionMember; + } + + public void setSqlExpression(com.linkedin.feathr.compute.SqlExpression value) { + checkNotNull(); + super._map.clear(); + _sqlExpressionMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.SqlExpression", value.data()); + } + + public static SlidingWindowFeature.GroupBy.ProjectionMask createMask() { + return new SlidingWindowFeature.GroupBy.ProjectionMask(); + } + + @Override + public SlidingWindowFeature.GroupBy clone() + throws CloneNotSupportedException + { + SlidingWindowFeature.GroupBy __clone = ((SlidingWindowFeature.GroupBy) super.clone()); + __clone.__changeListener = new SlidingWindowFeature.GroupBy.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public SlidingWindowFeature.GroupBy copy() + throws CloneNotSupportedException + { + SlidingWindowFeature.GroupBy __copy = ((SlidingWindowFeature.GroupBy) super.copy()); + __copy._sqlExpressionMember = null; + __copy.__changeListener = new SlidingWindowFeature.GroupBy.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final SlidingWindowFeature.GroupBy __objectRef; + + private ChangeListener(SlidingWindowFeature.GroupBy reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "com.linkedin.feathr.compute.SqlExpression": + __objectRef._sqlExpressionMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.compute.SqlExpression.Fields SqlExpression() { + return new com.linkedin.feathr.compute.SqlExpression.Fields(getPathComponents(), "com.linkedin.feathr.compute.SqlExpression"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.SqlExpression.ProjectionMask _SqlExpressionMask; + + ProjectionMask() { + super(2); + } + + public SlidingWindowFeature.GroupBy.ProjectionMask withSqlExpression(Function nestedMask) { + _SqlExpressionMask = nestedMask.apply(((_SqlExpressionMask == null)?com.linkedin.feathr.compute.SqlExpression.createMask():_SqlExpressionMask)); + getDataMap().put("com.linkedin.feathr.compute.SqlExpression", _SqlExpressionMask.getDataMap()); + return this; + } + + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.SlidingWindowFeature.TargetColumn.ProjectionMask _targetColumnMask; + private com.linkedin.feathr.compute.Window.ProjectionMask _windowMask; + private com.linkedin.feathr.compute.LateralViewArray.ProjectionMask _lateralViewsMask; + private com.linkedin.feathr.compute.SlidingWindowFeature.Filter.ProjectionMask _filterMask; + private com.linkedin.feathr.compute.SlidingWindowFeature.GroupBy.ProjectionMask _groupByMask; + + ProjectionMask() { + super(10); + } + + /** + * The target column to perform aggregation against. + * + */ + public SlidingWindowFeature.ProjectionMask withTargetColumn(Function nestedMask) { + _targetColumnMask = nestedMask.apply(((_targetColumnMask == null)?SlidingWindowFeature.TargetColumn.createMask():_targetColumnMask)); + getDataMap().put("targetColumn", _targetColumnMask.getDataMap()); + return this; + } + + /** + * The target column to perform aggregation against. + * + */ + public SlidingWindowFeature.ProjectionMask withTargetColumn() { + _targetColumnMask = null; + getDataMap().put("targetColumn", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Represents supported types of aggregation. + * + */ + public SlidingWindowFeature.ProjectionMask withAggregationType() { + getDataMap().put("aggregationType", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Represents the time window to look back from label data's timestamp. + * + */ + public SlidingWindowFeature.ProjectionMask withWindow(Function nestedMask) { + _windowMask = nestedMask.apply(((_windowMask == null)?Window.createMask():_windowMask)); + getDataMap().put("window", _windowMask.getDataMap()); + return this; + } + + /** + * Represents the time window to look back from label data's timestamp. + * + */ + public SlidingWindowFeature.ProjectionMask withWindow() { + _windowMask = null; + getDataMap().put("window", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details. + * + */ + public SlidingWindowFeature.ProjectionMask withLateralViews(Function nestedMask) { + _lateralViewsMask = nestedMask.apply(((_lateralViewsMask == null)?LateralViewArray.createMask():_lateralViewsMask)); + getDataMap().put("lateralViews", _lateralViewsMask.getDataMap()); + return this; + } + + /** + * Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details. + * + */ + public SlidingWindowFeature.ProjectionMask withLateralViews() { + _lateralViewsMask = null; + getDataMap().put("lateralViews", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details. + * + */ + public SlidingWindowFeature.ProjectionMask withLateralViews(Function nestedMask, Integer start, Integer count) { + _lateralViewsMask = nestedMask.apply(((_lateralViewsMask == null)?LateralViewArray.createMask():_lateralViewsMask)); + getDataMap().put("lateralViews", _lateralViewsMask.getDataMap()); + if (start!= null) { + getDataMap().getDataMap("lateralViews").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("lateralViews").put("$count", count); + } + return this; + } + + /** + * Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details. + * + */ + public SlidingWindowFeature.ProjectionMask withLateralViews(Integer start, Integer count) { + _lateralViewsMask = null; + getDataMap().put("lateralViews", new DataMap(3)); + if (start!= null) { + getDataMap().getDataMap("lateralViews").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("lateralViews").put("$count", count); + } + return this; + } + + /** + * Represents the filter statement before the aggregation. + * + */ + public SlidingWindowFeature.ProjectionMask withFilter(Function nestedMask) { + _filterMask = nestedMask.apply(((_filterMask == null)?SlidingWindowFeature.Filter.createMask():_filterMask)); + getDataMap().put("filter", _filterMask.getDataMap()); + return this; + } + + /** + * Represents the filter statement before the aggregation. + * + */ + public SlidingWindowFeature.ProjectionMask withFilter() { + _filterMask = null; + getDataMap().put("filter", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Represents the target to be grouped by before aggregation. If groupBy is not set, the aggregation will be performed over the entire dataset. + * + */ + public SlidingWindowFeature.ProjectionMask withGroupBy(Function nestedMask) { + _groupByMask = nestedMask.apply(((_groupByMask == null)?SlidingWindowFeature.GroupBy.createMask():_groupByMask)); + getDataMap().put("groupBy", _groupByMask.getDataMap()); + return this; + } + + /** + * Represents the target to be grouped by before aggregation. If groupBy is not set, the aggregation will be performed over the entire dataset. + * + */ + public SlidingWindowFeature.ProjectionMask withGroupBy() { + _groupByMask = null; + getDataMap().put("groupBy", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Represents the max number of groups (with aggregation results) to return. + * + */ + public SlidingWindowFeature.ProjectionMask withLimit() { + getDataMap().put("limit", MaskMap.POSITIVE_MASK); + return this; + } + + } + + @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SlidingWindowFeature.pdl.") + public static class TargetColumn + extends UnionTemplate + { + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}}]", SchemaFormatType.PDL)); + private com.linkedin.feathr.compute.SqlExpression _sqlExpressionMember = null; + private SlidingWindowFeature.TargetColumn.ChangeListener __changeListener = new SlidingWindowFeature.TargetColumn.ChangeListener(this); + private final static DataSchema MEMBER_SqlExpression = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.SqlExpression"); + + public TargetColumn() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public TargetColumn(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static SlidingWindowFeature.TargetColumn create(com.linkedin.feathr.compute.SqlExpression value) { + SlidingWindowFeature.TargetColumn newUnion = new SlidingWindowFeature.TargetColumn(); + newUnion.setSqlExpression(value); + return newUnion; + } + + public boolean isSqlExpression() { + return memberIs("com.linkedin.feathr.compute.SqlExpression"); + } + + public com.linkedin.feathr.compute.SqlExpression getSqlExpression() { + checkNotNull(); + if (_sqlExpressionMember!= null) { + return _sqlExpressionMember; + } + Object __rawValue = super._map.get("com.linkedin.feathr.compute.SqlExpression"); + _sqlExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.SqlExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _sqlExpressionMember; + } + + public void setSqlExpression(com.linkedin.feathr.compute.SqlExpression value) { + checkNotNull(); + super._map.clear(); + _sqlExpressionMember = value; + CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.SqlExpression", value.data()); + } + + public static SlidingWindowFeature.TargetColumn.ProjectionMask createMask() { + return new SlidingWindowFeature.TargetColumn.ProjectionMask(); + } + + @Override + public SlidingWindowFeature.TargetColumn clone() + throws CloneNotSupportedException + { + SlidingWindowFeature.TargetColumn __clone = ((SlidingWindowFeature.TargetColumn) super.clone()); + __clone.__changeListener = new SlidingWindowFeature.TargetColumn.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public SlidingWindowFeature.TargetColumn copy() + throws CloneNotSupportedException + { + SlidingWindowFeature.TargetColumn __copy = ((SlidingWindowFeature.TargetColumn) super.copy()); + __copy._sqlExpressionMember = null; + __copy.__changeListener = new SlidingWindowFeature.TargetColumn.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final SlidingWindowFeature.TargetColumn __objectRef; + + private ChangeListener(SlidingWindowFeature.TargetColumn reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "com.linkedin.feathr.compute.SqlExpression": + __objectRef._sqlExpressionMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.compute.SqlExpression.Fields SqlExpression() { + return new com.linkedin.feathr.compute.SqlExpression.Fields(getPathComponents(), "com.linkedin.feathr.compute.SqlExpression"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.SqlExpression.ProjectionMask _SqlExpressionMask; + + ProjectionMask() { + super(2); + } + + public SlidingWindowFeature.TargetColumn.ProjectionMask withSqlExpression(Function nestedMask) { + _SqlExpressionMask = nestedMask.apply(((_SqlExpressionMask == null)?com.linkedin.feathr.compute.SqlExpression.createMask():_SqlExpressionMask)); + getDataMap().put("com.linkedin.feathr.compute.SqlExpression", _SqlExpressionMask.getDataMap()); + return this; + } + + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SqlExpression.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SqlExpression.java new file mode 100644 index 000000000..53e075683 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SqlExpression.java @@ -0,0 +1,260 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * An expression in Spark SQL. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SqlExpression.pdl.") +public class SqlExpression + extends RecordTemplate +{ + + private final static SqlExpression.Fields _fields = new SqlExpression.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}", SchemaFormatType.PDL)); + private String _sqlField = null; + private SqlExpression.ChangeListener __changeListener = new SqlExpression.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Sql = SCHEMA.getField("sql"); + + public SqlExpression() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public SqlExpression(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static SqlExpression.Fields fields() { + return _fields; + } + + public static SqlExpression.ProjectionMask createMask() { + return new SqlExpression.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for sql + * + * @see SqlExpression.Fields#sql + */ + public boolean hasSql() { + if (_sqlField!= null) { + return true; + } + return super._map.containsKey("sql"); + } + + /** + * Remover for sql + * + * @see SqlExpression.Fields#sql + */ + public void removeSql() { + super._map.remove("sql"); + } + + /** + * Getter for sql + * + * @see SqlExpression.Fields#sql + */ + public String getSql(GetMode mode) { + switch (mode) { + case STRICT: + return getSql(); + case DEFAULT: + case NULL: + if (_sqlField!= null) { + return _sqlField; + } else { + Object __rawValue = super._map.get("sql"); + _sqlField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _sqlField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for sql + * + * @return + * Required field. Could be null for partial record. + * @see SqlExpression.Fields#sql + */ + @Nonnull + public String getSql() { + if (_sqlField!= null) { + return _sqlField; + } else { + Object __rawValue = super._map.get("sql"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("sql"); + } + _sqlField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _sqlField; + } + } + + /** + * Setter for sql + * + * @see SqlExpression.Fields#sql + */ + public SqlExpression setSql(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setSql(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field sql of com.linkedin.feathr.compute.SqlExpression"); + } else { + CheckedUtil.putWithoutChecking(super._map, "sql", value); + _sqlField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeSql(); + } else { + CheckedUtil.putWithoutChecking(super._map, "sql", value); + _sqlField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "sql", value); + _sqlField = value; + } + break; + } + return this; + } + + /** + * Setter for sql + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see SqlExpression.Fields#sql + */ + public SqlExpression setSql( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field sql of com.linkedin.feathr.compute.SqlExpression to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "sql", value); + _sqlField = value; + } + return this; + } + + @Override + public SqlExpression clone() + throws CloneNotSupportedException + { + SqlExpression __clone = ((SqlExpression) super.clone()); + __clone.__changeListener = new SqlExpression.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public SqlExpression copy() + throws CloneNotSupportedException + { + SqlExpression __copy = ((SqlExpression) super.copy()); + __copy._sqlField = null; + __copy.__changeListener = new SqlExpression.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final SqlExpression __objectRef; + + private ChangeListener(SqlExpression reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "sql": + __objectRef._sqlField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The Spark SQL expression. + * + */ + public PathSpec sql() { + return new PathSpec(getPathComponents(), "sql"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(2); + } + + /** + * The Spark SQL expression. + * + */ + public SqlExpression.ProjectionMask withSql() { + getDataMap().put("sql", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorCategory.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorCategory.java new file mode 100644 index 000000000..6fc37fcc4 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorCategory.java @@ -0,0 +1,42 @@ + +package com.linkedin.feathr.compute; + +import javax.annotation.Generated; +import com.linkedin.data.schema.EnumDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; + + +/** + * Supported Tensor categories in feathr and Quince. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\TensorCategory.pdl.") +public enum TensorCategory { + + + /** + * Dense tensors store values in a contiguous sequential block of memory where all values are represented. + * + */ + DENSE, + + /** + * Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them. + * + */ + SPARSE, + + /** + * Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions. + * + */ + RAGGED, + $UNKNOWN; + private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}", SchemaFormatType.PDL)); + + public static EnumDataSchema dataSchema() { + return SCHEMA; + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorFeatureFormat.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorFeatureFormat.java new file mode 100644 index 000000000..a869fdbbc --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorFeatureFormat.java @@ -0,0 +1,603 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\TensorFeatureFormat.pdl.") +public class TensorFeatureFormat + extends RecordTemplate +{ + + private final static TensorFeatureFormat.Fields _fields = new TensorFeatureFormat.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}", SchemaFormatType.PDL)); + private TensorCategory _tensorCategoryField = null; + private ValueType _valueTypeField = null; + private DimensionArray _dimensionsField = null; + private TensorFeatureFormat.ChangeListener __changeListener = new TensorFeatureFormat.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_TensorCategory = SCHEMA.getField("tensorCategory"); + private final static RecordDataSchema.Field FIELD_ValueType = SCHEMA.getField("valueType"); + private final static RecordDataSchema.Field FIELD_Dimensions = SCHEMA.getField("dimensions"); + + public TensorFeatureFormat() { + super(new DataMap(4, 0.75F), SCHEMA, 2); + addChangeListener(__changeListener); + } + + public TensorFeatureFormat(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static TensorFeatureFormat.Fields fields() { + return _fields; + } + + public static TensorFeatureFormat.ProjectionMask createMask() { + return new TensorFeatureFormat.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for tensorCategory + * + * @see TensorFeatureFormat.Fields#tensorCategory + */ + public boolean hasTensorCategory() { + if (_tensorCategoryField!= null) { + return true; + } + return super._map.containsKey("tensorCategory"); + } + + /** + * Remover for tensorCategory + * + * @see TensorFeatureFormat.Fields#tensorCategory + */ + public void removeTensorCategory() { + super._map.remove("tensorCategory"); + } + + /** + * Getter for tensorCategory + * + * @see TensorFeatureFormat.Fields#tensorCategory + */ + public TensorCategory getTensorCategory(GetMode mode) { + switch (mode) { + case STRICT: + return getTensorCategory(); + case DEFAULT: + case NULL: + if (_tensorCategoryField!= null) { + return _tensorCategoryField; + } else { + Object __rawValue = super._map.get("tensorCategory"); + _tensorCategoryField = DataTemplateUtil.coerceEnumOutput(__rawValue, TensorCategory.class, TensorCategory.$UNKNOWN); + return _tensorCategoryField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for tensorCategory + * + * @return + * Required field. Could be null for partial record. + * @see TensorFeatureFormat.Fields#tensorCategory + */ + @Nonnull + public TensorCategory getTensorCategory() { + if (_tensorCategoryField!= null) { + return _tensorCategoryField; + } else { + Object __rawValue = super._map.get("tensorCategory"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("tensorCategory"); + } + _tensorCategoryField = DataTemplateUtil.coerceEnumOutput(__rawValue, TensorCategory.class, TensorCategory.$UNKNOWN); + return _tensorCategoryField; + } + } + + /** + * Setter for tensorCategory + * + * @see TensorFeatureFormat.Fields#tensorCategory + */ + public TensorFeatureFormat setTensorCategory(TensorCategory value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setTensorCategory(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field tensorCategory of com.linkedin.feathr.compute.TensorFeatureFormat"); + } else { + CheckedUtil.putWithoutChecking(super._map, "tensorCategory", value.name()); + _tensorCategoryField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeTensorCategory(); + } else { + CheckedUtil.putWithoutChecking(super._map, "tensorCategory", value.name()); + _tensorCategoryField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "tensorCategory", value.name()); + _tensorCategoryField = value; + } + break; + } + return this; + } + + /** + * Setter for tensorCategory + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TensorFeatureFormat.Fields#tensorCategory + */ + public TensorFeatureFormat setTensorCategory( + @Nonnull + TensorCategory value) { + if (value == null) { + throw new NullPointerException("Cannot set field tensorCategory of com.linkedin.feathr.compute.TensorFeatureFormat to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "tensorCategory", value.name()); + _tensorCategoryField = value; + } + return this; + } + + /** + * Existence checker for valueType + * + * @see TensorFeatureFormat.Fields#valueType + */ + public boolean hasValueType() { + if (_valueTypeField!= null) { + return true; + } + return super._map.containsKey("valueType"); + } + + /** + * Remover for valueType + * + * @see TensorFeatureFormat.Fields#valueType + */ + public void removeValueType() { + super._map.remove("valueType"); + } + + /** + * Getter for valueType + * + * @see TensorFeatureFormat.Fields#valueType + */ + public ValueType getValueType(GetMode mode) { + switch (mode) { + case STRICT: + return getValueType(); + case DEFAULT: + case NULL: + if (_valueTypeField!= null) { + return _valueTypeField; + } else { + Object __rawValue = super._map.get("valueType"); + _valueTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, ValueType.class, ValueType.$UNKNOWN); + return _valueTypeField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for valueType + * + * @return + * Required field. Could be null for partial record. + * @see TensorFeatureFormat.Fields#valueType + */ + @Nonnull + public ValueType getValueType() { + if (_valueTypeField!= null) { + return _valueTypeField; + } else { + Object __rawValue = super._map.get("valueType"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("valueType"); + } + _valueTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, ValueType.class, ValueType.$UNKNOWN); + return _valueTypeField; + } + } + + /** + * Setter for valueType + * + * @see TensorFeatureFormat.Fields#valueType + */ + public TensorFeatureFormat setValueType(ValueType value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setValueType(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field valueType of com.linkedin.feathr.compute.TensorFeatureFormat"); + } else { + CheckedUtil.putWithoutChecking(super._map, "valueType", value.name()); + _valueTypeField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeValueType(); + } else { + CheckedUtil.putWithoutChecking(super._map, "valueType", value.name()); + _valueTypeField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "valueType", value.name()); + _valueTypeField = value; + } + break; + } + return this; + } + + /** + * Setter for valueType + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TensorFeatureFormat.Fields#valueType + */ + public TensorFeatureFormat setValueType( + @Nonnull + ValueType value) { + if (value == null) { + throw new NullPointerException("Cannot set field valueType of com.linkedin.feathr.compute.TensorFeatureFormat to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "valueType", value.name()); + _valueTypeField = value; + } + return this; + } + + /** + * Existence checker for dimensions + * + * @see TensorFeatureFormat.Fields#dimensions + */ + public boolean hasDimensions() { + if (_dimensionsField!= null) { + return true; + } + return super._map.containsKey("dimensions"); + } + + /** + * Remover for dimensions + * + * @see TensorFeatureFormat.Fields#dimensions + */ + public void removeDimensions() { + super._map.remove("dimensions"); + } + + /** + * Getter for dimensions + * + * @see TensorFeatureFormat.Fields#dimensions + */ + public DimensionArray getDimensions(GetMode mode) { + switch (mode) { + case STRICT: + return getDimensions(); + case DEFAULT: + case NULL: + if (_dimensionsField!= null) { + return _dimensionsField; + } else { + Object __rawValue = super._map.get("dimensions"); + _dimensionsField = ((__rawValue == null)?null:new DimensionArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _dimensionsField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for dimensions + * + * @return + * Required field. Could be null for partial record. + * @see TensorFeatureFormat.Fields#dimensions + */ + @Nonnull + public DimensionArray getDimensions() { + if (_dimensionsField!= null) { + return _dimensionsField; + } else { + Object __rawValue = super._map.get("dimensions"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("dimensions"); + } + _dimensionsField = ((__rawValue == null)?null:new DimensionArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _dimensionsField; + } + } + + /** + * Setter for dimensions + * + * @see TensorFeatureFormat.Fields#dimensions + */ + public TensorFeatureFormat setDimensions(DimensionArray value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setDimensions(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field dimensions of com.linkedin.feathr.compute.TensorFeatureFormat"); + } else { + CheckedUtil.putWithoutChecking(super._map, "dimensions", value.data()); + _dimensionsField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeDimensions(); + } else { + CheckedUtil.putWithoutChecking(super._map, "dimensions", value.data()); + _dimensionsField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "dimensions", value.data()); + _dimensionsField = value; + } + break; + } + return this; + } + + /** + * Setter for dimensions + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TensorFeatureFormat.Fields#dimensions + */ + public TensorFeatureFormat setDimensions( + @Nonnull + DimensionArray value) { + if (value == null) { + throw new NullPointerException("Cannot set field dimensions of com.linkedin.feathr.compute.TensorFeatureFormat to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "dimensions", value.data()); + _dimensionsField = value; + } + return this; + } + + @Override + public TensorFeatureFormat clone() + throws CloneNotSupportedException + { + TensorFeatureFormat __clone = ((TensorFeatureFormat) super.clone()); + __clone.__changeListener = new TensorFeatureFormat.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public TensorFeatureFormat copy() + throws CloneNotSupportedException + { + TensorFeatureFormat __copy = ((TensorFeatureFormat) super.copy()); + __copy._tensorCategoryField = null; + __copy._valueTypeField = null; + __copy._dimensionsField = null; + __copy.__changeListener = new TensorFeatureFormat.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final TensorFeatureFormat __objectRef; + + private ChangeListener(TensorFeatureFormat reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "tensorCategory": + __objectRef._tensorCategoryField = null; + break; + case "valueType": + __objectRef._valueTypeField = null; + break; + case "dimensions": + __objectRef._dimensionsField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Type of the tensor, for example, dense tensor. + * + */ + public PathSpec tensorCategory() { + return new PathSpec(getPathComponents(), "tensorCategory"); + } + + /** + * Type of the value column. + * + */ + public PathSpec valueType() { + return new PathSpec(getPathComponents(), "valueType"); + } + + /** + * A feature data can have zero or more dimensions (columns that represent keys). + * + */ + public com.linkedin.feathr.compute.DimensionArray.Fields dimensions() { + return new com.linkedin.feathr.compute.DimensionArray.Fields(getPathComponents(), "dimensions"); + } + + /** + * A feature data can have zero or more dimensions (columns that represent keys). + * + */ + public PathSpec dimensions(Integer start, Integer count) { + PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "dimensions"); + if (start!= null) { + arrayPathSpec.setAttribute("start", start); + } + if (count!= null) { + arrayPathSpec.setAttribute("count", count); + } + return arrayPathSpec; + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.DimensionArray.ProjectionMask _dimensionsMask; + + ProjectionMask() { + super(4); + } + + /** + * Type of the tensor, for example, dense tensor. + * + */ + public TensorFeatureFormat.ProjectionMask withTensorCategory() { + getDataMap().put("tensorCategory", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Type of the value column. + * + */ + public TensorFeatureFormat.ProjectionMask withValueType() { + getDataMap().put("valueType", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * A feature data can have zero or more dimensions (columns that represent keys). + * + */ + public TensorFeatureFormat.ProjectionMask withDimensions(Function nestedMask) { + _dimensionsMask = nestedMask.apply(((_dimensionsMask == null)?DimensionArray.createMask():_dimensionsMask)); + getDataMap().put("dimensions", _dimensionsMask.getDataMap()); + return this; + } + + /** + * A feature data can have zero or more dimensions (columns that represent keys). + * + */ + public TensorFeatureFormat.ProjectionMask withDimensions() { + _dimensionsMask = null; + getDataMap().put("dimensions", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * A feature data can have zero or more dimensions (columns that represent keys). + * + */ + public TensorFeatureFormat.ProjectionMask withDimensions(Function nestedMask, Integer start, Integer count) { + _dimensionsMask = nestedMask.apply(((_dimensionsMask == null)?DimensionArray.createMask():_dimensionsMask)); + getDataMap().put("dimensions", _dimensionsMask.getDataMap()); + if (start!= null) { + getDataMap().getDataMap("dimensions").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("dimensions").put("$count", count); + } + return this; + } + + /** + * A feature data can have zero or more dimensions (columns that represent keys). + * + */ + public TensorFeatureFormat.ProjectionMask withDimensions(Integer start, Integer count) { + _dimensionsMask = null; + getDataMap().put("dimensions", new DataMap(3)); + if (start!= null) { + getDataMap().getDataMap("dimensions").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("dimensions").put("$count", count); + } + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Time.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Time.java new file mode 100644 index 000000000..d00b4db2d --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Time.java @@ -0,0 +1,30 @@ + +package com.linkedin.feathr.compute; + +import javax.annotation.Generated; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.TyperefDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.TyperefInfo; + + +/** + * Number of milliseconds since midnight, January 1, 1970 UTC. It must be a positive number + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Time.pdl.") +public class Time + extends TyperefInfo +{ + + private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Number of milliseconds since midnight, January 1, 1970 UTC. It must be a positive number*/@compliance=\"NONE\"typeref Time=long", SchemaFormatType.PDL)); + + public Time() { + super(SCHEMA); + } + + public static TyperefDataSchema dataSchema() { + return SCHEMA; + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TimestampCol.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TimestampCol.java new file mode 100644 index 000000000..b884b7ac7 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TimestampCol.java @@ -0,0 +1,401 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * Representation of a timestamp column field + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\TimestampCol.pdl.") +public class TimestampCol + extends RecordTemplate +{ + + private final static TimestampCol.Fields _fields = new TimestampCol.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Representation of a timestamp column field*/record TimestampCol{/**Timestamp column expression.*/expression:string/**Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis*/format:string}", SchemaFormatType.PDL)); + private String _expressionField = null; + private String _formatField = null; + private TimestampCol.ChangeListener __changeListener = new TimestampCol.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Expression = SCHEMA.getField("expression"); + private final static RecordDataSchema.Field FIELD_Format = SCHEMA.getField("format"); + + public TimestampCol() { + super(new DataMap(3, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public TimestampCol(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static TimestampCol.Fields fields() { + return _fields; + } + + public static TimestampCol.ProjectionMask createMask() { + return new TimestampCol.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for expression + * + * @see TimestampCol.Fields#expression + */ + public boolean hasExpression() { + if (_expressionField!= null) { + return true; + } + return super._map.containsKey("expression"); + } + + /** + * Remover for expression + * + * @see TimestampCol.Fields#expression + */ + public void removeExpression() { + super._map.remove("expression"); + } + + /** + * Getter for expression + * + * @see TimestampCol.Fields#expression + */ + public String getExpression(GetMode mode) { + switch (mode) { + case STRICT: + return getExpression(); + case DEFAULT: + case NULL: + if (_expressionField!= null) { + return _expressionField; + } else { + Object __rawValue = super._map.get("expression"); + _expressionField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _expressionField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for expression + * + * @return + * Required field. Could be null for partial record. + * @see TimestampCol.Fields#expression + */ + @Nonnull + public String getExpression() { + if (_expressionField!= null) { + return _expressionField; + } else { + Object __rawValue = super._map.get("expression"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("expression"); + } + _expressionField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _expressionField; + } + } + + /** + * Setter for expression + * + * @see TimestampCol.Fields#expression + */ + public TimestampCol setExpression(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setExpression(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field expression of com.linkedin.feathr.compute.TimestampCol"); + } else { + CheckedUtil.putWithoutChecking(super._map, "expression", value); + _expressionField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeExpression(); + } else { + CheckedUtil.putWithoutChecking(super._map, "expression", value); + _expressionField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "expression", value); + _expressionField = value; + } + break; + } + return this; + } + + /** + * Setter for expression + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TimestampCol.Fields#expression + */ + public TimestampCol setExpression( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field expression of com.linkedin.feathr.compute.TimestampCol to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "expression", value); + _expressionField = value; + } + return this; + } + + /** + * Existence checker for format + * + * @see TimestampCol.Fields#format + */ + public boolean hasFormat() { + if (_formatField!= null) { + return true; + } + return super._map.containsKey("format"); + } + + /** + * Remover for format + * + * @see TimestampCol.Fields#format + */ + public void removeFormat() { + super._map.remove("format"); + } + + /** + * Getter for format + * + * @see TimestampCol.Fields#format + */ + public String getFormat(GetMode mode) { + switch (mode) { + case STRICT: + return getFormat(); + case DEFAULT: + case NULL: + if (_formatField!= null) { + return _formatField; + } else { + Object __rawValue = super._map.get("format"); + _formatField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _formatField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for format + * + * @return + * Required field. Could be null for partial record. + * @see TimestampCol.Fields#format + */ + @Nonnull + public String getFormat() { + if (_formatField!= null) { + return _formatField; + } else { + Object __rawValue = super._map.get("format"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("format"); + } + _formatField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _formatField; + } + } + + /** + * Setter for format + * + * @see TimestampCol.Fields#format + */ + public TimestampCol setFormat(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFormat(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field format of com.linkedin.feathr.compute.TimestampCol"); + } else { + CheckedUtil.putWithoutChecking(super._map, "format", value); + _formatField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeFormat(); + } else { + CheckedUtil.putWithoutChecking(super._map, "format", value); + _formatField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "format", value); + _formatField = value; + } + break; + } + return this; + } + + /** + * Setter for format + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TimestampCol.Fields#format + */ + public TimestampCol setFormat( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field format of com.linkedin.feathr.compute.TimestampCol to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "format", value); + _formatField = value; + } + return this; + } + + @Override + public TimestampCol clone() + throws CloneNotSupportedException + { + TimestampCol __clone = ((TimestampCol) super.clone()); + __clone.__changeListener = new TimestampCol.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public TimestampCol copy() + throws CloneNotSupportedException + { + TimestampCol __copy = ((TimestampCol) super.copy()); + __copy._expressionField = null; + __copy._formatField = null; + __copy.__changeListener = new TimestampCol.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final TimestampCol __objectRef; + + private ChangeListener(TimestampCol reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "expression": + __objectRef._expressionField = null; + break; + case "format": + __objectRef._formatField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Timestamp column expression. + * + */ + public PathSpec expression() { + return new PathSpec(getPathComponents(), "expression"); + } + + /** + * Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis + * + */ + public PathSpec format() { + return new PathSpec(getPathComponents(), "format"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(3); + } + + /** + * Timestamp column expression. + * + */ + public TimestampCol.ProjectionMask withExpression() { + getDataMap().put("expression", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis + * + */ + public TimestampCol.ProjectionMask withFormat() { + getDataMap().put("format", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Transformation.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Transformation.java new file mode 100644 index 000000000..56de17b55 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Transformation.java @@ -0,0 +1,1065 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * Representation of a transformation node. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Transformation.pdl.") +public class Transformation + extends RecordTemplate +{ + + private final static Transformation.Fields _fields = new Transformation.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Representation of a transformation node.*/record Transformation includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**An array of node references which should be considered as input to apply the transformation function.*/inputs:array[/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}]/**The transformation function.*/function:/**The transformation function*/record TransformationFunction{/**Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class.\r\n*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**The various attributes required to represent the transformation function are captured in a map format.\r\nFor example, mvel expression or java udf class name*/parameters:optional map[string,string]}/**Feature name here is used so we retain feature name, type, and default values even after graph is resolved.\r\nFeature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}", SchemaFormatType.PDL)); + private Integer _idField = null; + private ConcreteKey _concreteKeyField = null; + private NodeReferenceArray _inputsField = null; + private TransformationFunction _functionField = null; + private String _featureNameField = null; + private FeatureVersion _featureVersionField = null; + private Transformation.ChangeListener __changeListener = new Transformation.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); + private final static RecordDataSchema.Field FIELD_ConcreteKey = SCHEMA.getField("concreteKey"); + private final static RecordDataSchema.Field FIELD_Inputs = SCHEMA.getField("inputs"); + private final static RecordDataSchema.Field FIELD_Function = SCHEMA.getField("function"); + private final static RecordDataSchema.Field FIELD_FeatureName = SCHEMA.getField("featureName"); + private final static RecordDataSchema.Field FIELD_FeatureVersion = SCHEMA.getField("featureVersion"); + + public Transformation() { + super(new DataMap(8, 0.75F), SCHEMA, 6); + addChangeListener(__changeListener); + } + + public Transformation(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static Transformation.Fields fields() { + return _fields; + } + + public static Transformation.ProjectionMask createMask() { + return new Transformation.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for id + * + * @see Transformation.Fields#id + */ + public boolean hasId() { + if (_idField!= null) { + return true; + } + return super._map.containsKey("id"); + } + + /** + * Remover for id + * + * @see Transformation.Fields#id + */ + public void removeId() { + super._map.remove("id"); + } + + /** + * Getter for id + * + * @see Transformation.Fields#id + */ + public Integer getId(GetMode mode) { + switch (mode) { + case STRICT: + return getId(); + case DEFAULT: + case NULL: + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for id + * + * @return + * Required field. Could be null for partial record. + * @see Transformation.Fields#id + */ + @Nonnull + public Integer getId() { + if (_idField!= null) { + return _idField; + } else { + Object __rawValue = super._map.get("id"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("id"); + } + _idField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _idField; + } + } + + /** + * Setter for id + * + * @see Transformation.Fields#id + */ + public Transformation setId(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setId(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.Transformation"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeId(); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + break; + } + return this; + } + + /** + * Setter for id + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Transformation.Fields#id + */ + public Transformation setId( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.Transformation to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + } + return this; + } + + /** + * Setter for id + * + * @see Transformation.Fields#id + */ + public Transformation setId(int value) { + CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); + _idField = value; + return this; + } + + /** + * Existence checker for concreteKey + * + * @see Transformation.Fields#concreteKey + */ + public boolean hasConcreteKey() { + if (_concreteKeyField!= null) { + return true; + } + return super._map.containsKey("concreteKey"); + } + + /** + * Remover for concreteKey + * + * @see Transformation.Fields#concreteKey + */ + public void removeConcreteKey() { + super._map.remove("concreteKey"); + } + + /** + * Getter for concreteKey + * + * @see Transformation.Fields#concreteKey + */ + public ConcreteKey getConcreteKey(GetMode mode) { + return getConcreteKey(); + } + + /** + * Getter for concreteKey + * + * @return + * Optional field. Always check for null. + * @see Transformation.Fields#concreteKey + */ + @Nullable + public ConcreteKey getConcreteKey() { + if (_concreteKeyField!= null) { + return _concreteKeyField; + } else { + Object __rawValue = super._map.get("concreteKey"); + _concreteKeyField = ((__rawValue == null)?null:new ConcreteKey(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _concreteKeyField; + } + } + + /** + * Setter for concreteKey + * + * @see Transformation.Fields#concreteKey + */ + public Transformation setConcreteKey(ConcreteKey value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setConcreteKey(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeConcreteKey(); + } else { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + break; + } + return this; + } + + /** + * Setter for concreteKey + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Transformation.Fields#concreteKey + */ + public Transformation setConcreteKey( + @Nonnull + ConcreteKey value) { + if (value == null) { + throw new NullPointerException("Cannot set field concreteKey of com.linkedin.feathr.compute.Transformation to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); + _concreteKeyField = value; + } + return this; + } + + /** + * Existence checker for inputs + * + * @see Transformation.Fields#inputs + */ + public boolean hasInputs() { + if (_inputsField!= null) { + return true; + } + return super._map.containsKey("inputs"); + } + + /** + * Remover for inputs + * + * @see Transformation.Fields#inputs + */ + public void removeInputs() { + super._map.remove("inputs"); + } + + /** + * Getter for inputs + * + * @see Transformation.Fields#inputs + */ + public NodeReferenceArray getInputs(GetMode mode) { + switch (mode) { + case STRICT: + return getInputs(); + case DEFAULT: + case NULL: + if (_inputsField!= null) { + return _inputsField; + } else { + Object __rawValue = super._map.get("inputs"); + _inputsField = ((__rawValue == null)?null:new NodeReferenceArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _inputsField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for inputs + * + * @return + * Required field. Could be null for partial record. + * @see Transformation.Fields#inputs + */ + @Nonnull + public NodeReferenceArray getInputs() { + if (_inputsField!= null) { + return _inputsField; + } else { + Object __rawValue = super._map.get("inputs"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("inputs"); + } + _inputsField = ((__rawValue == null)?null:new NodeReferenceArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _inputsField; + } + } + + /** + * Setter for inputs + * + * @see Transformation.Fields#inputs + */ + public Transformation setInputs(NodeReferenceArray value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setInputs(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field inputs of com.linkedin.feathr.compute.Transformation"); + } else { + CheckedUtil.putWithoutChecking(super._map, "inputs", value.data()); + _inputsField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeInputs(); + } else { + CheckedUtil.putWithoutChecking(super._map, "inputs", value.data()); + _inputsField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "inputs", value.data()); + _inputsField = value; + } + break; + } + return this; + } + + /** + * Setter for inputs + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Transformation.Fields#inputs + */ + public Transformation setInputs( + @Nonnull + NodeReferenceArray value) { + if (value == null) { + throw new NullPointerException("Cannot set field inputs of com.linkedin.feathr.compute.Transformation to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "inputs", value.data()); + _inputsField = value; + } + return this; + } + + /** + * Existence checker for function + * + * @see Transformation.Fields#function + */ + public boolean hasFunction() { + if (_functionField!= null) { + return true; + } + return super._map.containsKey("function"); + } + + /** + * Remover for function + * + * @see Transformation.Fields#function + */ + public void removeFunction() { + super._map.remove("function"); + } + + /** + * Getter for function + * + * @see Transformation.Fields#function + */ + public TransformationFunction getFunction(GetMode mode) { + switch (mode) { + case STRICT: + return getFunction(); + case DEFAULT: + case NULL: + if (_functionField!= null) { + return _functionField; + } else { + Object __rawValue = super._map.get("function"); + _functionField = ((__rawValue == null)?null:new TransformationFunction(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _functionField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for function + * + * @return + * Required field. Could be null for partial record. + * @see Transformation.Fields#function + */ + @Nonnull + public TransformationFunction getFunction() { + if (_functionField!= null) { + return _functionField; + } else { + Object __rawValue = super._map.get("function"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("function"); + } + _functionField = ((__rawValue == null)?null:new TransformationFunction(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _functionField; + } + } + + /** + * Setter for function + * + * @see Transformation.Fields#function + */ + public Transformation setFunction(TransformationFunction value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFunction(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field function of com.linkedin.feathr.compute.Transformation"); + } else { + CheckedUtil.putWithoutChecking(super._map, "function", value.data()); + _functionField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeFunction(); + } else { + CheckedUtil.putWithoutChecking(super._map, "function", value.data()); + _functionField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "function", value.data()); + _functionField = value; + } + break; + } + return this; + } + + /** + * Setter for function + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Transformation.Fields#function + */ + public Transformation setFunction( + @Nonnull + TransformationFunction value) { + if (value == null) { + throw new NullPointerException("Cannot set field function of com.linkedin.feathr.compute.Transformation to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "function", value.data()); + _functionField = value; + } + return this; + } + + /** + * Existence checker for featureName + * + * @see Transformation.Fields#featureName + */ + public boolean hasFeatureName() { + if (_featureNameField!= null) { + return true; + } + return super._map.containsKey("featureName"); + } + + /** + * Remover for featureName + * + * @see Transformation.Fields#featureName + */ + public void removeFeatureName() { + super._map.remove("featureName"); + } + + /** + * Getter for featureName + * + * @see Transformation.Fields#featureName + */ + public String getFeatureName(GetMode mode) { + switch (mode) { + case STRICT: + return getFeatureName(); + case DEFAULT: + case NULL: + if (_featureNameField!= null) { + return _featureNameField; + } else { + Object __rawValue = super._map.get("featureName"); + _featureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _featureNameField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for featureName + * + * @return + * Required field. Could be null for partial record. + * @see Transformation.Fields#featureName + */ + @Nonnull + public String getFeatureName() { + if (_featureNameField!= null) { + return _featureNameField; + } else { + Object __rawValue = super._map.get("featureName"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("featureName"); + } + _featureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _featureNameField; + } + } + + /** + * Setter for featureName + * + * @see Transformation.Fields#featureName + */ + public Transformation setFeatureName(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFeatureName(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field featureName of com.linkedin.feathr.compute.Transformation"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureName", value); + _featureNameField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeFeatureName(); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureName", value); + _featureNameField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "featureName", value); + _featureNameField = value; + } + break; + } + return this; + } + + /** + * Setter for featureName + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Transformation.Fields#featureName + */ + public Transformation setFeatureName( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field featureName of com.linkedin.feathr.compute.Transformation to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureName", value); + _featureNameField = value; + } + return this; + } + + /** + * Existence checker for featureVersion + * + * @see Transformation.Fields#featureVersion + */ + public boolean hasFeatureVersion() { + if (_featureVersionField!= null) { + return true; + } + return super._map.containsKey("featureVersion"); + } + + /** + * Remover for featureVersion + * + * @see Transformation.Fields#featureVersion + */ + public void removeFeatureVersion() { + super._map.remove("featureVersion"); + } + + /** + * Getter for featureVersion + * + * @see Transformation.Fields#featureVersion + */ + public FeatureVersion getFeatureVersion(GetMode mode) { + switch (mode) { + case STRICT: + return getFeatureVersion(); + case DEFAULT: + case NULL: + if (_featureVersionField!= null) { + return _featureVersionField; + } else { + Object __rawValue = super._map.get("featureVersion"); + _featureVersionField = ((__rawValue == null)?null:new FeatureVersion(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _featureVersionField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for featureVersion + * + * @return + * Required field. Could be null for partial record. + * @see Transformation.Fields#featureVersion + */ + @Nonnull + public FeatureVersion getFeatureVersion() { + if (_featureVersionField!= null) { + return _featureVersionField; + } else { + Object __rawValue = super._map.get("featureVersion"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("featureVersion"); + } + _featureVersionField = ((__rawValue == null)?null:new FeatureVersion(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _featureVersionField; + } + } + + /** + * Setter for featureVersion + * + * @see Transformation.Fields#featureVersion + */ + public Transformation setFeatureVersion(FeatureVersion value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFeatureVersion(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field featureVersion of com.linkedin.feathr.compute.Transformation"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); + _featureVersionField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeFeatureVersion(); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); + _featureVersionField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); + _featureVersionField = value; + } + break; + } + return this; + } + + /** + * Setter for featureVersion + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Transformation.Fields#featureVersion + */ + public Transformation setFeatureVersion( + @Nonnull + FeatureVersion value) { + if (value == null) { + throw new NullPointerException("Cannot set field featureVersion of com.linkedin.feathr.compute.Transformation to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); + _featureVersionField = value; + } + return this; + } + + @Override + public Transformation clone() + throws CloneNotSupportedException + { + Transformation __clone = ((Transformation) super.clone()); + __clone.__changeListener = new Transformation.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public Transformation copy() + throws CloneNotSupportedException + { + Transformation __copy = ((Transformation) super.copy()); + __copy._featureNameField = null; + __copy._inputsField = null; + __copy._functionField = null; + __copy._idField = null; + __copy._concreteKeyField = null; + __copy._featureVersionField = null; + __copy.__changeListener = new Transformation.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final Transformation __objectRef; + + private ChangeListener(Transformation reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "featureName": + __objectRef._featureNameField = null; + break; + case "inputs": + __objectRef._inputsField = null; + break; + case "function": + __objectRef._functionField = null; + break; + case "id": + __objectRef._idField = null; + break; + case "concreteKey": + __objectRef._concreteKeyField = null; + break; + case "featureVersion": + __objectRef._featureVersionField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The node would be represented by this id. + * + */ + public PathSpec id() { + return new PathSpec(getPathComponents(), "id"); + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public com.linkedin.feathr.compute.ConcreteKey.Fields concreteKey() { + return new com.linkedin.feathr.compute.ConcreteKey.Fields(getPathComponents(), "concreteKey"); + } + + /** + * An array of node references which should be considered as input to apply the transformation function. + * + */ + public com.linkedin.feathr.compute.NodeReferenceArray.Fields inputs() { + return new com.linkedin.feathr.compute.NodeReferenceArray.Fields(getPathComponents(), "inputs"); + } + + /** + * An array of node references which should be considered as input to apply the transformation function. + * + */ + public PathSpec inputs(Integer start, Integer count) { + PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "inputs"); + if (start!= null) { + arrayPathSpec.setAttribute("start", start); + } + if (count!= null) { + arrayPathSpec.setAttribute("count", count); + } + return arrayPathSpec; + } + + /** + * The transformation function. + * + */ + public com.linkedin.feathr.compute.TransformationFunction.Fields function() { + return new com.linkedin.feathr.compute.TransformationFunction.Fields(getPathComponents(), "function"); + } + + /** + * Feature name here is used so we retain feature name, type, and default values even after graph is resolved. + * Feature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias. + * + */ + public PathSpec featureName() { + return new PathSpec(getPathComponents(), "featureName"); + } + + /** + * feature version of the feature + * + */ + public com.linkedin.feathr.compute.FeatureVersion.Fields featureVersion() { + return new com.linkedin.feathr.compute.FeatureVersion.Fields(getPathComponents(), "featureVersion"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.compute.ConcreteKey.ProjectionMask _concreteKeyMask; + private com.linkedin.feathr.compute.NodeReferenceArray.ProjectionMask _inputsMask; + private com.linkedin.feathr.compute.TransformationFunction.ProjectionMask _functionMask; + private com.linkedin.feathr.compute.FeatureVersion.ProjectionMask _featureVersionMask; + + ProjectionMask() { + super(8); + } + + /** + * The node would be represented by this id. + * + */ + public Transformation.ProjectionMask withId() { + getDataMap().put("id", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public Transformation.ProjectionMask withConcreteKey(Function nestedMask) { + _concreteKeyMask = nestedMask.apply(((_concreteKeyMask == null)?ConcreteKey.createMask():_concreteKeyMask)); + getDataMap().put("concreteKey", _concreteKeyMask.getDataMap()); + return this; + } + + /** + * The key for which this node is being requested. + * If this node is a Source node, the engine can use the key to fetch or join the feature. + * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but + * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, + * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) + * + */ + public Transformation.ProjectionMask withConcreteKey() { + _concreteKeyMask = null; + getDataMap().put("concreteKey", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * An array of node references which should be considered as input to apply the transformation function. + * + */ + public Transformation.ProjectionMask withInputs(Function nestedMask) { + _inputsMask = nestedMask.apply(((_inputsMask == null)?NodeReferenceArray.createMask():_inputsMask)); + getDataMap().put("inputs", _inputsMask.getDataMap()); + return this; + } + + /** + * An array of node references which should be considered as input to apply the transformation function. + * + */ + public Transformation.ProjectionMask withInputs() { + _inputsMask = null; + getDataMap().put("inputs", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * An array of node references which should be considered as input to apply the transformation function. + * + */ + public Transformation.ProjectionMask withInputs(Function nestedMask, Integer start, Integer count) { + _inputsMask = nestedMask.apply(((_inputsMask == null)?NodeReferenceArray.createMask():_inputsMask)); + getDataMap().put("inputs", _inputsMask.getDataMap()); + if (start!= null) { + getDataMap().getDataMap("inputs").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("inputs").put("$count", count); + } + return this; + } + + /** + * An array of node references which should be considered as input to apply the transformation function. + * + */ + public Transformation.ProjectionMask withInputs(Integer start, Integer count) { + _inputsMask = null; + getDataMap().put("inputs", new DataMap(3)); + if (start!= null) { + getDataMap().getDataMap("inputs").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("inputs").put("$count", count); + } + return this; + } + + /** + * The transformation function. + * + */ + public Transformation.ProjectionMask withFunction(Function nestedMask) { + _functionMask = nestedMask.apply(((_functionMask == null)?TransformationFunction.createMask():_functionMask)); + getDataMap().put("function", _functionMask.getDataMap()); + return this; + } + + /** + * The transformation function. + * + */ + public Transformation.ProjectionMask withFunction() { + _functionMask = null; + getDataMap().put("function", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Feature name here is used so we retain feature name, type, and default values even after graph is resolved. + * Feature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias. + * + */ + public Transformation.ProjectionMask withFeatureName() { + getDataMap().put("featureName", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * feature version of the feature + * + */ + public Transformation.ProjectionMask withFeatureVersion(Function nestedMask) { + _featureVersionMask = nestedMask.apply(((_featureVersionMask == null)?FeatureVersion.createMask():_featureVersionMask)); + getDataMap().put("featureVersion", _featureVersionMask.getDataMap()); + return this; + } + + /** + * feature version of the feature + * + */ + public Transformation.ProjectionMask withFeatureVersion() { + _featureVersionMask = null; + getDataMap().put("featureVersion", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TransformationFunction.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TransformationFunction.java new file mode 100644 index 000000000..1a44e0193 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TransformationFunction.java @@ -0,0 +1,384 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.StringMap; + + +/** + * The transformation function + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\TransformationFunction.pdl.") +public class TransformationFunction + extends RecordTemplate +{ + + private final static TransformationFunction.Fields _fields = new TransformationFunction.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**The transformation function*/record TransformationFunction{/**Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class.\r\n*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**The various attributes required to represent the transformation function are captured in a map format.\r\nFor example, mvel expression or java udf class name*/parameters:optional map[string,string]}", SchemaFormatType.PDL)); + private String _operatorField = null; + private StringMap _parametersField = null; + private TransformationFunction.ChangeListener __changeListener = new TransformationFunction.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Operator = SCHEMA.getField("operator"); + private final static RecordDataSchema.Field FIELD_Parameters = SCHEMA.getField("parameters"); + + public TransformationFunction() { + super(new DataMap(3, 0.75F), SCHEMA, 2); + addChangeListener(__changeListener); + } + + public TransformationFunction(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static TransformationFunction.Fields fields() { + return _fields; + } + + public static TransformationFunction.ProjectionMask createMask() { + return new TransformationFunction.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for operator + * + * @see TransformationFunction.Fields#operator + */ + public boolean hasOperator() { + if (_operatorField!= null) { + return true; + } + return super._map.containsKey("operator"); + } + + /** + * Remover for operator + * + * @see TransformationFunction.Fields#operator + */ + public void removeOperator() { + super._map.remove("operator"); + } + + /** + * Getter for operator + * + * @see TransformationFunction.Fields#operator + */ + public String getOperator(GetMode mode) { + switch (mode) { + case STRICT: + return getOperator(); + case DEFAULT: + case NULL: + if (_operatorField!= null) { + return _operatorField; + } else { + Object __rawValue = super._map.get("operator"); + _operatorField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _operatorField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for operator + * + * @return + * Required field. Could be null for partial record. + * @see TransformationFunction.Fields#operator + */ + @Nonnull + public String getOperator() { + if (_operatorField!= null) { + return _operatorField; + } else { + Object __rawValue = super._map.get("operator"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("operator"); + } + _operatorField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _operatorField; + } + } + + /** + * Setter for operator + * + * @see TransformationFunction.Fields#operator + */ + public TransformationFunction setOperator(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setOperator(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field operator of com.linkedin.feathr.compute.TransformationFunction"); + } else { + CheckedUtil.putWithoutChecking(super._map, "operator", value); + _operatorField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeOperator(); + } else { + CheckedUtil.putWithoutChecking(super._map, "operator", value); + _operatorField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "operator", value); + _operatorField = value; + } + break; + } + return this; + } + + /** + * Setter for operator + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TransformationFunction.Fields#operator + */ + public TransformationFunction setOperator( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field operator of com.linkedin.feathr.compute.TransformationFunction to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "operator", value); + _operatorField = value; + } + return this; + } + + /** + * Existence checker for parameters + * + * @see TransformationFunction.Fields#parameters + */ + public boolean hasParameters() { + if (_parametersField!= null) { + return true; + } + return super._map.containsKey("parameters"); + } + + /** + * Remover for parameters + * + * @see TransformationFunction.Fields#parameters + */ + public void removeParameters() { + super._map.remove("parameters"); + } + + /** + * Getter for parameters + * + * @see TransformationFunction.Fields#parameters + */ + public StringMap getParameters(GetMode mode) { + return getParameters(); + } + + /** + * Getter for parameters + * + * @return + * Optional field. Always check for null. + * @see TransformationFunction.Fields#parameters + */ + @Nullable + public StringMap getParameters() { + if (_parametersField!= null) { + return _parametersField; + } else { + Object __rawValue = super._map.get("parameters"); + _parametersField = ((__rawValue == null)?null:new StringMap(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _parametersField; + } + } + + /** + * Setter for parameters + * + * @see TransformationFunction.Fields#parameters + */ + public TransformationFunction setParameters(StringMap value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setParameters(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeParameters(); + } else { + CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); + _parametersField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); + _parametersField = value; + } + break; + } + return this; + } + + /** + * Setter for parameters + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TransformationFunction.Fields#parameters + */ + public TransformationFunction setParameters( + @Nonnull + StringMap value) { + if (value == null) { + throw new NullPointerException("Cannot set field parameters of com.linkedin.feathr.compute.TransformationFunction to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); + _parametersField = value; + } + return this; + } + + @Override + public TransformationFunction clone() + throws CloneNotSupportedException + { + TransformationFunction __clone = ((TransformationFunction) super.clone()); + __clone.__changeListener = new TransformationFunction.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public TransformationFunction copy() + throws CloneNotSupportedException + { + TransformationFunction __copy = ((TransformationFunction) super.copy()); + __copy._parametersField = null; + __copy._operatorField = null; + __copy.__changeListener = new TransformationFunction.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final TransformationFunction __objectRef; + + private ChangeListener(TransformationFunction reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "parameters": + __objectRef._parametersField = null; + break; + case "operator": + __objectRef._operatorField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class. + * + * + */ + public PathSpec operator() { + return new PathSpec(getPathComponents(), "operator"); + } + + /** + * The various attributes required to represent the transformation function are captured in a map format. + * For example, mvel expression or java udf class name + * + */ + public PathSpec parameters() { + return new PathSpec(getPathComponents(), "parameters"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(3); + } + + /** + * Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class. + * + * + */ + public TransformationFunction.ProjectionMask withOperator() { + getDataMap().put("operator", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * The various attributes required to represent the transformation function are captured in a map format. + * For example, mvel expression or java udf class name + * + */ + public TransformationFunction.ProjectionMask withParameters() { + getDataMap().put("parameters", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Unit.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Unit.java new file mode 100644 index 000000000..c433dc921 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Unit.java @@ -0,0 +1,48 @@ + +package com.linkedin.feathr.compute; + +import javax.annotation.Generated; +import com.linkedin.data.schema.EnumDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; + + +/** + * + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Window.pdl.") +public enum Unit { + + + /** + * A day. + * + */ + DAY, + + /** + * An hour. + * + */ + HOUR, + + /** + * A minute. + * + */ + MINUTE, + + /** + * A second. + * + */ + SECOND, + $UNKNOWN; + private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute,enum Unit{/** A day. */DAY/** An hour. */HOUR/** A minute. */MINUTE/** A second. */SECOND}", SchemaFormatType.PDL)); + + public static EnumDataSchema dataSchema() { + return SCHEMA; + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/UserDefinedFunction.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/UserDefinedFunction.java new file mode 100644 index 000000000..40035e4b8 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/UserDefinedFunction.java @@ -0,0 +1,407 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.StringMap; + + +/** + * User defined function that can be used in feature extraction or derivation. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\UserDefinedFunction.pdl.") +public class UserDefinedFunction + extends RecordTemplate +{ + + private final static UserDefinedFunction.Fields _fields = new UserDefinedFunction.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**User defined function that can be used in feature extraction or derivation.*/record UserDefinedFunction{/**Reference to the class that implements the user defined function.*/clazz:string/**Some UserDefinedFunction requires additional custom parameters. This field defines the custom parameters of the user defined function, represented as a map of string to json blob. The key is the parameter name, and the value is the parameter value represented as a json blob. For example, the parameters may look like: { param1 : [\"waterlooCompany_terms_hashed\", \"waterlooCompany_values\"], param2 : \"com.linkedin.quasar.encoding.SomeEncodingClass\u00e2\u20ac\ufffd } feathr will be responsible of parsing the parameters map into a CustomParameters class defined by application: public class CustomParameters { List param1; String param2; } CustomParameters will be used in the constructor of the UserDefinedFunction.*/parameters:map[string,string]={}}", SchemaFormatType.PDL)); + private String _clazzField = null; + private StringMap _parametersField = null; + private UserDefinedFunction.ChangeListener __changeListener = new UserDefinedFunction.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Clazz = SCHEMA.getField("clazz"); + private final static RecordDataSchema.Field FIELD_Parameters = SCHEMA.getField("parameters"); + private final static StringMap DEFAULT_Parameters; + + static { + DEFAULT_Parameters = ((FIELD_Parameters.getDefault() == null)?null:new StringMap(DataTemplateUtil.castOrThrow(FIELD_Parameters.getDefault(), DataMap.class))); + } + + public UserDefinedFunction() { + super(new DataMap(3, 0.75F), SCHEMA, 2); + addChangeListener(__changeListener); + } + + public UserDefinedFunction(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UserDefinedFunction.Fields fields() { + return _fields; + } + + public static UserDefinedFunction.ProjectionMask createMask() { + return new UserDefinedFunction.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for clazz + * + * @see UserDefinedFunction.Fields#clazz + */ + public boolean hasClazz() { + if (_clazzField!= null) { + return true; + } + return super._map.containsKey("clazz"); + } + + /** + * Remover for clazz + * + * @see UserDefinedFunction.Fields#clazz + */ + public void removeClazz() { + super._map.remove("clazz"); + } + + /** + * Getter for clazz + * + * @see UserDefinedFunction.Fields#clazz + */ + public String getClazz(GetMode mode) { + switch (mode) { + case STRICT: + return getClazz(); + case DEFAULT: + case NULL: + if (_clazzField!= null) { + return _clazzField; + } else { + Object __rawValue = super._map.get("clazz"); + _clazzField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _clazzField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for clazz + * + * @return + * Required field. Could be null for partial record. + * @see UserDefinedFunction.Fields#clazz + */ + @Nonnull + public String getClazz() { + if (_clazzField!= null) { + return _clazzField; + } else { + Object __rawValue = super._map.get("clazz"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("clazz"); + } + _clazzField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _clazzField; + } + } + + /** + * Setter for clazz + * + * @see UserDefinedFunction.Fields#clazz + */ + public UserDefinedFunction setClazz(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setClazz(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field clazz of com.linkedin.feathr.compute.UserDefinedFunction"); + } else { + CheckedUtil.putWithoutChecking(super._map, "clazz", value); + _clazzField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeClazz(); + } else { + CheckedUtil.putWithoutChecking(super._map, "clazz", value); + _clazzField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "clazz", value); + _clazzField = value; + } + break; + } + return this; + } + + /** + * Setter for clazz + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see UserDefinedFunction.Fields#clazz + */ + public UserDefinedFunction setClazz( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field clazz of com.linkedin.feathr.compute.UserDefinedFunction to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "clazz", value); + _clazzField = value; + } + return this; + } + + /** + * Existence checker for parameters + * + * @see UserDefinedFunction.Fields#parameters + */ + public boolean hasParameters() { + if (_parametersField!= null) { + return true; + } + return super._map.containsKey("parameters"); + } + + /** + * Remover for parameters + * + * @see UserDefinedFunction.Fields#parameters + */ + public void removeParameters() { + super._map.remove("parameters"); + } + + /** + * Getter for parameters + * + * @see UserDefinedFunction.Fields#parameters + */ + public StringMap getParameters(GetMode mode) { + switch (mode) { + case STRICT: + case DEFAULT: + return getParameters(); + case NULL: + if (_parametersField!= null) { + return _parametersField; + } else { + Object __rawValue = super._map.get("parameters"); + _parametersField = ((__rawValue == null)?null:new StringMap(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _parametersField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for parameters + * + * @return + * Required field. Could be null for partial record. + * @see UserDefinedFunction.Fields#parameters + */ + @Nonnull + public StringMap getParameters() { + if (_parametersField!= null) { + return _parametersField; + } else { + Object __rawValue = super._map.get("parameters"); + if (__rawValue == null) { + return DEFAULT_Parameters; + } + _parametersField = ((__rawValue == null)?null:new StringMap(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _parametersField; + } + } + + /** + * Setter for parameters + * + * @see UserDefinedFunction.Fields#parameters + */ + public UserDefinedFunction setParameters(StringMap value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setParameters(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field parameters of com.linkedin.feathr.compute.UserDefinedFunction"); + } else { + CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); + _parametersField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeParameters(); + } else { + CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); + _parametersField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); + _parametersField = value; + } + break; + } + return this; + } + + /** + * Setter for parameters + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see UserDefinedFunction.Fields#parameters + */ + public UserDefinedFunction setParameters( + @Nonnull + StringMap value) { + if (value == null) { + throw new NullPointerException("Cannot set field parameters of com.linkedin.feathr.compute.UserDefinedFunction to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); + _parametersField = value; + } + return this; + } + + @Override + public UserDefinedFunction clone() + throws CloneNotSupportedException + { + UserDefinedFunction __clone = ((UserDefinedFunction) super.clone()); + __clone.__changeListener = new UserDefinedFunction.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public UserDefinedFunction copy() + throws CloneNotSupportedException + { + UserDefinedFunction __copy = ((UserDefinedFunction) super.copy()); + __copy._clazzField = null; + __copy._parametersField = null; + __copy.__changeListener = new UserDefinedFunction.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final UserDefinedFunction __objectRef; + + private ChangeListener(UserDefinedFunction reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "clazz": + __objectRef._clazzField = null; + break; + case "parameters": + __objectRef._parametersField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Reference to the class that implements the user defined function. + * + */ + public PathSpec clazz() { + return new PathSpec(getPathComponents(), "clazz"); + } + + /** + * Some UserDefinedFunction requires additional custom parameters. This field defines the custom parameters of the user defined function, represented as a map of string to json blob. The key is the parameter name, and the value is the parameter value represented as a json blob. For example, the parameters may look like: { param1 : ["waterlooCompany_terms_hashed", "waterlooCompany_values"], param2 : "com.linkedin.quasar.encoding.SomeEncodingClassâ€\ufffd } feathr will be responsible of parsing the parameters map into a CustomParameters class defined by application: public class CustomParameters { List param1; String param2; } CustomParameters will be used in the constructor of the UserDefinedFunction. + * + */ + public PathSpec parameters() { + return new PathSpec(getPathComponents(), "parameters"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(3); + } + + /** + * Reference to the class that implements the user defined function. + * + */ + public UserDefinedFunction.ProjectionMask withClazz() { + getDataMap().put("clazz", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Some UserDefinedFunction requires additional custom parameters. This field defines the custom parameters of the user defined function, represented as a map of string to json blob. The key is the parameter name, and the value is the parameter value represented as a json blob. For example, the parameters may look like: { param1 : ["waterlooCompany_terms_hashed", "waterlooCompany_values"], param2 : "com.linkedin.quasar.encoding.SomeEncodingClassâ€\ufffd } feathr will be responsible of parsing the parameters map into a CustomParameters class defined by application: public class CustomParameters { List param1; String param2; } CustomParameters will be used in the constructor of the UserDefinedFunction. + * + */ + public UserDefinedFunction.ProjectionMask withParameters() { + getDataMap().put("parameters", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ValueType.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ValueType.java new file mode 100644 index 000000000..f29229bd9 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ValueType.java @@ -0,0 +1,66 @@ + +package com.linkedin.feathr.compute; + +import javax.annotation.Generated; +import com.linkedin.data.schema.EnumDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; + + +/** + * Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\ValueType.pdl.") +public enum ValueType { + + + /** + * Integer. + * + */ + INT, + + /** + * Long. + * + */ + LONG, + + /** + * Float. + * + */ + FLOAT, + + /** + * Double. + * + */ + DOUBLE, + + /** + * String. + * + */ + STRING, + + /** + * Boolean. + * + */ + BOOLEAN, + + /** + * Byte array. + * + */ + BYTES, + $UNKNOWN; + private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}", SchemaFormatType.PDL)); + + public static EnumDataSchema dataSchema() { + return SCHEMA; + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Window.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Window.java new file mode 100644 index 000000000..83512a2e7 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Window.java @@ -0,0 +1,412 @@ + +package com.linkedin.feathr.compute; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * Represents a time window used in sliding window algorithms. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Window.pdl.") +public class Window + extends RecordTemplate +{ + + private final static Window.Fields _fields = new Window.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Represents a time window used in sliding window algorithms.*/record Window{/**Represents the duration of the window.*/size:int/**Represents a unit of time.*/unit:enum Unit{/** A day. */DAY/** An hour. */HOUR/** A minute. */MINUTE/** A second. */SECOND}}", SchemaFormatType.PDL)); + private Integer _sizeField = null; + private Unit _unitField = null; + private Window.ChangeListener __changeListener = new Window.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Size = SCHEMA.getField("size"); + private final static RecordDataSchema.Field FIELD_Unit = SCHEMA.getField("unit"); + + public Window() { + super(new DataMap(3, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public Window(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static Window.Fields fields() { + return _fields; + } + + public static Window.ProjectionMask createMask() { + return new Window.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for size + * + * @see Window.Fields#size + */ + public boolean hasSize() { + if (_sizeField!= null) { + return true; + } + return super._map.containsKey("size"); + } + + /** + * Remover for size + * + * @see Window.Fields#size + */ + public void removeSize() { + super._map.remove("size"); + } + + /** + * Getter for size + * + * @see Window.Fields#size + */ + public Integer getSize(GetMode mode) { + switch (mode) { + case STRICT: + return getSize(); + case DEFAULT: + case NULL: + if (_sizeField!= null) { + return _sizeField; + } else { + Object __rawValue = super._map.get("size"); + _sizeField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _sizeField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for size + * + * @return + * Required field. Could be null for partial record. + * @see Window.Fields#size + */ + @Nonnull + public Integer getSize() { + if (_sizeField!= null) { + return _sizeField; + } else { + Object __rawValue = super._map.get("size"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("size"); + } + _sizeField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _sizeField; + } + } + + /** + * Setter for size + * + * @see Window.Fields#size + */ + public Window setSize(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setSize(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field size of com.linkedin.feathr.compute.Window"); + } else { + CheckedUtil.putWithoutChecking(super._map, "size", DataTemplateUtil.coerceIntInput(value)); + _sizeField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeSize(); + } else { + CheckedUtil.putWithoutChecking(super._map, "size", DataTemplateUtil.coerceIntInput(value)); + _sizeField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "size", DataTemplateUtil.coerceIntInput(value)); + _sizeField = value; + } + break; + } + return this; + } + + /** + * Setter for size + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Window.Fields#size + */ + public Window setSize( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field size of com.linkedin.feathr.compute.Window to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "size", DataTemplateUtil.coerceIntInput(value)); + _sizeField = value; + } + return this; + } + + /** + * Setter for size + * + * @see Window.Fields#size + */ + public Window setSize(int value) { + CheckedUtil.putWithoutChecking(super._map, "size", DataTemplateUtil.coerceIntInput(value)); + _sizeField = value; + return this; + } + + /** + * Existence checker for unit + * + * @see Window.Fields#unit + */ + public boolean hasUnit() { + if (_unitField!= null) { + return true; + } + return super._map.containsKey("unit"); + } + + /** + * Remover for unit + * + * @see Window.Fields#unit + */ + public void removeUnit() { + super._map.remove("unit"); + } + + /** + * Getter for unit + * + * @see Window.Fields#unit + */ + public Unit getUnit(GetMode mode) { + switch (mode) { + case STRICT: + return getUnit(); + case DEFAULT: + case NULL: + if (_unitField!= null) { + return _unitField; + } else { + Object __rawValue = super._map.get("unit"); + _unitField = DataTemplateUtil.coerceEnumOutput(__rawValue, Unit.class, Unit.$UNKNOWN); + return _unitField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for unit + * + * @return + * Required field. Could be null for partial record. + * @see Window.Fields#unit + */ + @Nonnull + public Unit getUnit() { + if (_unitField!= null) { + return _unitField; + } else { + Object __rawValue = super._map.get("unit"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("unit"); + } + _unitField = DataTemplateUtil.coerceEnumOutput(__rawValue, Unit.class, Unit.$UNKNOWN); + return _unitField; + } + } + + /** + * Setter for unit + * + * @see Window.Fields#unit + */ + public Window setUnit(Unit value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setUnit(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field unit of com.linkedin.feathr.compute.Window"); + } else { + CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); + _unitField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeUnit(); + } else { + CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); + _unitField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); + _unitField = value; + } + break; + } + return this; + } + + /** + * Setter for unit + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Window.Fields#unit + */ + public Window setUnit( + @Nonnull + Unit value) { + if (value == null) { + throw new NullPointerException("Cannot set field unit of com.linkedin.feathr.compute.Window to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); + _unitField = value; + } + return this; + } + + @Override + public Window clone() + throws CloneNotSupportedException + { + Window __clone = ((Window) super.clone()); + __clone.__changeListener = new Window.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public Window copy() + throws CloneNotSupportedException + { + Window __copy = ((Window) super.copy()); + __copy._unitField = null; + __copy._sizeField = null; + __copy.__changeListener = new Window.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final Window __objectRef; + + private ChangeListener(Window reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "unit": + __objectRef._unitField = null; + break; + case "size": + __objectRef._sizeField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Represents the duration of the window. + * + */ + public PathSpec size() { + return new PathSpec(getPathComponents(), "size"); + } + + /** + * Represents a unit of time. + * + */ + public PathSpec unit() { + return new PathSpec(getPathComponents(), "unit"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(3); + } + + /** + * Represents the duration of the window. + * + */ + public Window.ProjectionMask withSize() { + getDataMap().put("size", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Represents a unit of time. + * + */ + public Window.ProjectionMask withUnit() { + getDataMap().put("unit", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteDateRange.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteDateRange.java new file mode 100644 index 000000000..f949423f5 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteDateRange.java @@ -0,0 +1,432 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * The absolute date range with start and end date being required fields. + * It accepts a start date and an end date which should be specifiied using the [[Date.pdl]] class. + * absoluteDateRange: { + * startDate: Date(day=1, month=1, year=2020) + * endDate: Date(day=3, month=1, year=2020) + * } + * In this case, the endDate > startDate. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\AbsoluteDateRange.pdl.") +public class AbsoluteDateRange + extends RecordTemplate +{ + + private final static AbsoluteDateRange.Fields _fields = new AbsoluteDateRange.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The absolute date range with start and end date being required fields.\r\nIt accepts a start date and an end date which should be specifiied using the [[Date.pdl]] class.\r\nabsoluteDateRange: {\r\n startDate: Date(day=1, month=1, year=2020)\r\n endDate: Date(day=3, month=1, year=2020)\r\n }\r\n In this case, the endDate > startDate.*/record AbsoluteDateRange{/**start date of the date range, with the start date included in the range.*/startDate:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}/**end date of the date range, with the end date included in the range.*/endDate:Date}", SchemaFormatType.PDL)); + private Date _startDateField = null; + private Date _endDateField = null; + private AbsoluteDateRange.ChangeListener __changeListener = new AbsoluteDateRange.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_StartDate = SCHEMA.getField("startDate"); + private final static RecordDataSchema.Field FIELD_EndDate = SCHEMA.getField("endDate"); + + public AbsoluteDateRange() { + super(new DataMap(3, 0.75F), SCHEMA, 3); + addChangeListener(__changeListener); + } + + public AbsoluteDateRange(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static AbsoluteDateRange.Fields fields() { + return _fields; + } + + public static AbsoluteDateRange.ProjectionMask createMask() { + return new AbsoluteDateRange.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for startDate + * + * @see AbsoluteDateRange.Fields#startDate + */ + public boolean hasStartDate() { + if (_startDateField!= null) { + return true; + } + return super._map.containsKey("startDate"); + } + + /** + * Remover for startDate + * + * @see AbsoluteDateRange.Fields#startDate + */ + public void removeStartDate() { + super._map.remove("startDate"); + } + + /** + * Getter for startDate + * + * @see AbsoluteDateRange.Fields#startDate + */ + public Date getStartDate(GetMode mode) { + switch (mode) { + case STRICT: + return getStartDate(); + case DEFAULT: + case NULL: + if (_startDateField!= null) { + return _startDateField; + } else { + Object __rawValue = super._map.get("startDate"); + _startDateField = ((__rawValue == null)?null:new Date(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _startDateField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for startDate + * + * @return + * Required field. Could be null for partial record. + * @see AbsoluteDateRange.Fields#startDate + */ + @Nonnull + public Date getStartDate() { + if (_startDateField!= null) { + return _startDateField; + } else { + Object __rawValue = super._map.get("startDate"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("startDate"); + } + _startDateField = ((__rawValue == null)?null:new Date(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _startDateField; + } + } + + /** + * Setter for startDate + * + * @see AbsoluteDateRange.Fields#startDate + */ + public AbsoluteDateRange setStartDate(Date value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setStartDate(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field startDate of com.linkedin.feathr.config.join.AbsoluteDateRange"); + } else { + CheckedUtil.putWithoutChecking(super._map, "startDate", value.data()); + _startDateField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeStartDate(); + } else { + CheckedUtil.putWithoutChecking(super._map, "startDate", value.data()); + _startDateField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "startDate", value.data()); + _startDateField = value; + } + break; + } + return this; + } + + /** + * Setter for startDate + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see AbsoluteDateRange.Fields#startDate + */ + public AbsoluteDateRange setStartDate( + @Nonnull + Date value) { + if (value == null) { + throw new NullPointerException("Cannot set field startDate of com.linkedin.feathr.config.join.AbsoluteDateRange to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "startDate", value.data()); + _startDateField = value; + } + return this; + } + + /** + * Existence checker for endDate + * + * @see AbsoluteDateRange.Fields#endDate + */ + public boolean hasEndDate() { + if (_endDateField!= null) { + return true; + } + return super._map.containsKey("endDate"); + } + + /** + * Remover for endDate + * + * @see AbsoluteDateRange.Fields#endDate + */ + public void removeEndDate() { + super._map.remove("endDate"); + } + + /** + * Getter for endDate + * + * @see AbsoluteDateRange.Fields#endDate + */ + public Date getEndDate(GetMode mode) { + switch (mode) { + case STRICT: + return getEndDate(); + case DEFAULT: + case NULL: + if (_endDateField!= null) { + return _endDateField; + } else { + Object __rawValue = super._map.get("endDate"); + _endDateField = ((__rawValue == null)?null:new Date(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _endDateField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for endDate + * + * @return + * Required field. Could be null for partial record. + * @see AbsoluteDateRange.Fields#endDate + */ + @Nonnull + public Date getEndDate() { + if (_endDateField!= null) { + return _endDateField; + } else { + Object __rawValue = super._map.get("endDate"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("endDate"); + } + _endDateField = ((__rawValue == null)?null:new Date(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _endDateField; + } + } + + /** + * Setter for endDate + * + * @see AbsoluteDateRange.Fields#endDate + */ + public AbsoluteDateRange setEndDate(Date value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setEndDate(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field endDate of com.linkedin.feathr.config.join.AbsoluteDateRange"); + } else { + CheckedUtil.putWithoutChecking(super._map, "endDate", value.data()); + _endDateField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeEndDate(); + } else { + CheckedUtil.putWithoutChecking(super._map, "endDate", value.data()); + _endDateField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "endDate", value.data()); + _endDateField = value; + } + break; + } + return this; + } + + /** + * Setter for endDate + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see AbsoluteDateRange.Fields#endDate + */ + public AbsoluteDateRange setEndDate( + @Nonnull + Date value) { + if (value == null) { + throw new NullPointerException("Cannot set field endDate of com.linkedin.feathr.config.join.AbsoluteDateRange to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "endDate", value.data()); + _endDateField = value; + } + return this; + } + + @Override + public AbsoluteDateRange clone() + throws CloneNotSupportedException + { + AbsoluteDateRange __clone = ((AbsoluteDateRange) super.clone()); + __clone.__changeListener = new AbsoluteDateRange.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public AbsoluteDateRange copy() + throws CloneNotSupportedException + { + AbsoluteDateRange __copy = ((AbsoluteDateRange) super.copy()); + __copy._endDateField = null; + __copy._startDateField = null; + __copy.__changeListener = new AbsoluteDateRange.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final AbsoluteDateRange __objectRef; + + private ChangeListener(AbsoluteDateRange reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "endDate": + __objectRef._endDateField = null; + break; + case "startDate": + __objectRef._startDateField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * start date of the date range, with the start date included in the range. + * + */ + public com.linkedin.feathr.config.join.Date.Fields startDate() { + return new com.linkedin.feathr.config.join.Date.Fields(getPathComponents(), "startDate"); + } + + /** + * end date of the date range, with the end date included in the range. + * + */ + public com.linkedin.feathr.config.join.Date.Fields endDate() { + return new com.linkedin.feathr.config.join.Date.Fields(getPathComponents(), "endDate"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.Date.ProjectionMask _startDateMask; + private com.linkedin.feathr.config.join.Date.ProjectionMask _endDateMask; + + ProjectionMask() { + super(3); + } + + /** + * start date of the date range, with the start date included in the range. + * + */ + public AbsoluteDateRange.ProjectionMask withStartDate(Function nestedMask) { + _startDateMask = nestedMask.apply(((_startDateMask == null)?Date.createMask():_startDateMask)); + getDataMap().put("startDate", _startDateMask.getDataMap()); + return this; + } + + /** + * start date of the date range, with the start date included in the range. + * + */ + public AbsoluteDateRange.ProjectionMask withStartDate() { + _startDateMask = null; + getDataMap().put("startDate", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * end date of the date range, with the end date included in the range. + * + */ + public AbsoluteDateRange.ProjectionMask withEndDate(Function nestedMask) { + _endDateMask = nestedMask.apply(((_endDateMask == null)?Date.createMask():_endDateMask)); + getDataMap().put("endDate", _endDateMask.getDataMap()); + return this; + } + + /** + * end date of the date range, with the end date included in the range. + * + */ + public AbsoluteDateRange.ProjectionMask withEndDate() { + _endDateMask = null; + getDataMap().put("endDate", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteTimeRange.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteTimeRange.java new file mode 100644 index 000000000..d51ec4573 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteTimeRange.java @@ -0,0 +1,802 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.DataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.UnionDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.UnionTemplate; + + +/** + * The absolute time range with start and end time being required fields. + * It accepts a start time and an end time which should be specifiied using the [[Date.pdl]] or the [[HourTime.pdl]] class. + * This model can be used to represent time range in daily or hourly interval. + * absoluteTimeRange: { + * startTime: TimeHour(day=1, month=1, year=2020, hour=13) + * endTime: TimeHour(day=3, month=1, year=2020, hour=2) + * } + * (or) + * absoluteTimeRange: { + * startTime: Date(day=1, month=1, year=2020) + * endTime: Date(day=3, month=1, year=2020) + * } + * endTime and startTime should always have the same granularity, ie - Daily or Hourly. + * endTme > startTime + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\AbsoluteTimeRange.pdl.") +public class AbsoluteTimeRange + extends RecordTemplate +{ + + private final static AbsoluteTimeRange.Fields _fields = new AbsoluteTimeRange.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The absolute time range with start and end time being required fields.\r\nIt accepts a start time and an end time which should be specifiied using the [[Date.pdl]] or the [[HourTime.pdl]] class.\r\nThis model can be used to represent time range in daily or hourly interval.\r\nabsoluteTimeRange: {\r\n startTime: TimeHour(day=1, month=1, year=2020, hour=13)\r\n endTime: TimeHour(day=3, month=1, year=2020, hour=2)\r\n }\r\n(or)\r\nabsoluteTimeRange: {\r\n startTime: Date(day=1, month=1, year=2020)\r\n endTime: Date(day=3, month=1, year=2020)\r\n }\r\nendTime and startTime should always have the same granularity, ie - Daily or Hourly.\r\nendTme > startTime*/record AbsoluteTimeRange{/**start time of the date range, in daily or hourly format with the start date included in the range.*/startTime:union[date:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}hourTime:/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}]/**end date of the date range, in daily or hourly format with the end date included in the range.*/endTime:union[date:Date,hourTime:HourTime]}", SchemaFormatType.PDL)); + private AbsoluteTimeRange.StartTime _startTimeField = null; + private AbsoluteTimeRange.EndTime _endTimeField = null; + private AbsoluteTimeRange.ChangeListener __changeListener = new AbsoluteTimeRange.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_StartTime = SCHEMA.getField("startTime"); + private final static RecordDataSchema.Field FIELD_EndTime = SCHEMA.getField("endTime"); + + public AbsoluteTimeRange() { + super(new DataMap(3, 0.75F), SCHEMA, 3); + addChangeListener(__changeListener); + } + + public AbsoluteTimeRange(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static AbsoluteTimeRange.Fields fields() { + return _fields; + } + + public static AbsoluteTimeRange.ProjectionMask createMask() { + return new AbsoluteTimeRange.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for startTime + * + * @see AbsoluteTimeRange.Fields#startTime + */ + public boolean hasStartTime() { + if (_startTimeField!= null) { + return true; + } + return super._map.containsKey("startTime"); + } + + /** + * Remover for startTime + * + * @see AbsoluteTimeRange.Fields#startTime + */ + public void removeStartTime() { + super._map.remove("startTime"); + } + + /** + * Getter for startTime + * + * @see AbsoluteTimeRange.Fields#startTime + */ + public AbsoluteTimeRange.StartTime getStartTime(GetMode mode) { + switch (mode) { + case STRICT: + return getStartTime(); + case DEFAULT: + case NULL: + if (_startTimeField!= null) { + return _startTimeField; + } else { + Object __rawValue = super._map.get("startTime"); + _startTimeField = ((__rawValue == null)?null:new AbsoluteTimeRange.StartTime(__rawValue)); + return _startTimeField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for startTime + * + * @return + * Required field. Could be null for partial record. + * @see AbsoluteTimeRange.Fields#startTime + */ + @Nonnull + public AbsoluteTimeRange.StartTime getStartTime() { + if (_startTimeField!= null) { + return _startTimeField; + } else { + Object __rawValue = super._map.get("startTime"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("startTime"); + } + _startTimeField = ((__rawValue == null)?null:new AbsoluteTimeRange.StartTime(__rawValue)); + return _startTimeField; + } + } + + /** + * Setter for startTime + * + * @see AbsoluteTimeRange.Fields#startTime + */ + public AbsoluteTimeRange setStartTime(AbsoluteTimeRange.StartTime value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setStartTime(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field startTime of com.linkedin.feathr.config.join.AbsoluteTimeRange"); + } else { + CheckedUtil.putWithoutChecking(super._map, "startTime", value.data()); + _startTimeField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeStartTime(); + } else { + CheckedUtil.putWithoutChecking(super._map, "startTime", value.data()); + _startTimeField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "startTime", value.data()); + _startTimeField = value; + } + break; + } + return this; + } + + /** + * Setter for startTime + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see AbsoluteTimeRange.Fields#startTime + */ + public AbsoluteTimeRange setStartTime( + @Nonnull + AbsoluteTimeRange.StartTime value) { + if (value == null) { + throw new NullPointerException("Cannot set field startTime of com.linkedin.feathr.config.join.AbsoluteTimeRange to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "startTime", value.data()); + _startTimeField = value; + } + return this; + } + + /** + * Existence checker for endTime + * + * @see AbsoluteTimeRange.Fields#endTime + */ + public boolean hasEndTime() { + if (_endTimeField!= null) { + return true; + } + return super._map.containsKey("endTime"); + } + + /** + * Remover for endTime + * + * @see AbsoluteTimeRange.Fields#endTime + */ + public void removeEndTime() { + super._map.remove("endTime"); + } + + /** + * Getter for endTime + * + * @see AbsoluteTimeRange.Fields#endTime + */ + public AbsoluteTimeRange.EndTime getEndTime(GetMode mode) { + switch (mode) { + case STRICT: + return getEndTime(); + case DEFAULT: + case NULL: + if (_endTimeField!= null) { + return _endTimeField; + } else { + Object __rawValue = super._map.get("endTime"); + _endTimeField = ((__rawValue == null)?null:new AbsoluteTimeRange.EndTime(__rawValue)); + return _endTimeField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for endTime + * + * @return + * Required field. Could be null for partial record. + * @see AbsoluteTimeRange.Fields#endTime + */ + @Nonnull + public AbsoluteTimeRange.EndTime getEndTime() { + if (_endTimeField!= null) { + return _endTimeField; + } else { + Object __rawValue = super._map.get("endTime"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("endTime"); + } + _endTimeField = ((__rawValue == null)?null:new AbsoluteTimeRange.EndTime(__rawValue)); + return _endTimeField; + } + } + + /** + * Setter for endTime + * + * @see AbsoluteTimeRange.Fields#endTime + */ + public AbsoluteTimeRange setEndTime(AbsoluteTimeRange.EndTime value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setEndTime(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field endTime of com.linkedin.feathr.config.join.AbsoluteTimeRange"); + } else { + CheckedUtil.putWithoutChecking(super._map, "endTime", value.data()); + _endTimeField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeEndTime(); + } else { + CheckedUtil.putWithoutChecking(super._map, "endTime", value.data()); + _endTimeField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "endTime", value.data()); + _endTimeField = value; + } + break; + } + return this; + } + + /** + * Setter for endTime + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see AbsoluteTimeRange.Fields#endTime + */ + public AbsoluteTimeRange setEndTime( + @Nonnull + AbsoluteTimeRange.EndTime value) { + if (value == null) { + throw new NullPointerException("Cannot set field endTime of com.linkedin.feathr.config.join.AbsoluteTimeRange to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "endTime", value.data()); + _endTimeField = value; + } + return this; + } + + @Override + public AbsoluteTimeRange clone() + throws CloneNotSupportedException + { + AbsoluteTimeRange __clone = ((AbsoluteTimeRange) super.clone()); + __clone.__changeListener = new AbsoluteTimeRange.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public AbsoluteTimeRange copy() + throws CloneNotSupportedException + { + AbsoluteTimeRange __copy = ((AbsoluteTimeRange) super.copy()); + __copy._startTimeField = null; + __copy._endTimeField = null; + __copy.__changeListener = new AbsoluteTimeRange.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final AbsoluteTimeRange __objectRef; + + private ChangeListener(AbsoluteTimeRange reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "startTime": + __objectRef._startTimeField = null; + break; + case "endTime": + __objectRef._endTimeField = null; + break; + } + } + + } + + @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\AbsoluteTimeRange.pdl.") + public static class EndTime + extends UnionTemplate + { + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[date:{namespace com.linkedin.feathr.config.join/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}}hourTime:{namespace com.linkedin.feathr.config.join/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}}]", SchemaFormatType.PDL)); + private com.linkedin.feathr.config.join.Date _dateMember = null; + private com.linkedin.feathr.config.join.HourTime _hourTimeMember = null; + private AbsoluteTimeRange.EndTime.ChangeListener __changeListener = new AbsoluteTimeRange.EndTime.ChangeListener(this); + private final static DataSchema MEMBER_Date = SCHEMA.getTypeByMemberKey("date"); + private final static DataSchema MEMBER_HourTime = SCHEMA.getTypeByMemberKey("hourTime"); + + public EndTime() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public EndTime(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static AbsoluteTimeRange.EndTime createWithDate(com.linkedin.feathr.config.join.Date value) { + AbsoluteTimeRange.EndTime newUnion = new AbsoluteTimeRange.EndTime(); + newUnion.setDate(value); + return newUnion; + } + + public boolean isDate() { + return memberIs("date"); + } + + public com.linkedin.feathr.config.join.Date getDate() { + checkNotNull(); + if (_dateMember!= null) { + return _dateMember; + } + Object __rawValue = super._map.get("date"); + _dateMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.Date(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _dateMember; + } + + public void setDate(com.linkedin.feathr.config.join.Date value) { + checkNotNull(); + super._map.clear(); + _dateMember = value; + CheckedUtil.putWithoutChecking(super._map, "date", value.data()); + } + + public static AbsoluteTimeRange.EndTime createWithHourTime(com.linkedin.feathr.config.join.HourTime value) { + AbsoluteTimeRange.EndTime newUnion = new AbsoluteTimeRange.EndTime(); + newUnion.setHourTime(value); + return newUnion; + } + + public boolean isHourTime() { + return memberIs("hourTime"); + } + + public com.linkedin.feathr.config.join.HourTime getHourTime() { + checkNotNull(); + if (_hourTimeMember!= null) { + return _hourTimeMember; + } + Object __rawValue = super._map.get("hourTime"); + _hourTimeMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.HourTime(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _hourTimeMember; + } + + public void setHourTime(com.linkedin.feathr.config.join.HourTime value) { + checkNotNull(); + super._map.clear(); + _hourTimeMember = value; + CheckedUtil.putWithoutChecking(super._map, "hourTime", value.data()); + } + + public static AbsoluteTimeRange.EndTime.ProjectionMask createMask() { + return new AbsoluteTimeRange.EndTime.ProjectionMask(); + } + + @Override + public AbsoluteTimeRange.EndTime clone() + throws CloneNotSupportedException + { + AbsoluteTimeRange.EndTime __clone = ((AbsoluteTimeRange.EndTime) super.clone()); + __clone.__changeListener = new AbsoluteTimeRange.EndTime.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public AbsoluteTimeRange.EndTime copy() + throws CloneNotSupportedException + { + AbsoluteTimeRange.EndTime __copy = ((AbsoluteTimeRange.EndTime) super.copy()); + __copy._dateMember = null; + __copy._hourTimeMember = null; + __copy.__changeListener = new AbsoluteTimeRange.EndTime.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final AbsoluteTimeRange.EndTime __objectRef; + + private ChangeListener(AbsoluteTimeRange.EndTime reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "date": + __objectRef._dateMember = null; + break; + case "hourTime": + __objectRef._hourTimeMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.config.join.Date.Fields Date() { + return new com.linkedin.feathr.config.join.Date.Fields(getPathComponents(), "date"); + } + + public com.linkedin.feathr.config.join.HourTime.Fields HourTime() { + return new com.linkedin.feathr.config.join.HourTime.Fields(getPathComponents(), "hourTime"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.Date.ProjectionMask _DateMask; + private com.linkedin.feathr.config.join.HourTime.ProjectionMask _HourTimeMask; + + ProjectionMask() { + super(3); + } + + public AbsoluteTimeRange.EndTime.ProjectionMask withDate(Function nestedMask) { + _DateMask = nestedMask.apply(((_DateMask == null)?com.linkedin.feathr.config.join.Date.createMask():_DateMask)); + getDataMap().put("date", _DateMask.getDataMap()); + return this; + } + + public AbsoluteTimeRange.EndTime.ProjectionMask withHourTime(Function nestedMask) { + _HourTimeMask = nestedMask.apply(((_HourTimeMask == null)?com.linkedin.feathr.config.join.HourTime.createMask():_HourTimeMask)); + getDataMap().put("hourTime", _HourTimeMask.getDataMap()); + return this; + } + + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * start time of the date range, in daily or hourly format with the start date included in the range. + * + */ + public com.linkedin.feathr.config.join.AbsoluteTimeRange.StartTime.Fields startTime() { + return new com.linkedin.feathr.config.join.AbsoluteTimeRange.StartTime.Fields(getPathComponents(), "startTime"); + } + + /** + * end date of the date range, in daily or hourly format with the end date included in the range. + * + */ + public com.linkedin.feathr.config.join.AbsoluteTimeRange.EndTime.Fields endTime() { + return new com.linkedin.feathr.config.join.AbsoluteTimeRange.EndTime.Fields(getPathComponents(), "endTime"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.AbsoluteTimeRange.StartTime.ProjectionMask _startTimeMask; + private com.linkedin.feathr.config.join.AbsoluteTimeRange.EndTime.ProjectionMask _endTimeMask; + + ProjectionMask() { + super(3); + } + + /** + * start time of the date range, in daily or hourly format with the start date included in the range. + * + */ + public AbsoluteTimeRange.ProjectionMask withStartTime(Function nestedMask) { + _startTimeMask = nestedMask.apply(((_startTimeMask == null)?AbsoluteTimeRange.StartTime.createMask():_startTimeMask)); + getDataMap().put("startTime", _startTimeMask.getDataMap()); + return this; + } + + /** + * start time of the date range, in daily or hourly format with the start date included in the range. + * + */ + public AbsoluteTimeRange.ProjectionMask withStartTime() { + _startTimeMask = null; + getDataMap().put("startTime", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * end date of the date range, in daily or hourly format with the end date included in the range. + * + */ + public AbsoluteTimeRange.ProjectionMask withEndTime(Function nestedMask) { + _endTimeMask = nestedMask.apply(((_endTimeMask == null)?AbsoluteTimeRange.EndTime.createMask():_endTimeMask)); + getDataMap().put("endTime", _endTimeMask.getDataMap()); + return this; + } + + /** + * end date of the date range, in daily or hourly format with the end date included in the range. + * + */ + public AbsoluteTimeRange.ProjectionMask withEndTime() { + _endTimeMask = null; + getDataMap().put("endTime", MaskMap.POSITIVE_MASK); + return this; + } + + } + + @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\AbsoluteTimeRange.pdl.") + public static class StartTime + extends UnionTemplate + { + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[date:{namespace com.linkedin.feathr.config.join/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}}hourTime:{namespace com.linkedin.feathr.config.join/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}}]", SchemaFormatType.PDL)); + private com.linkedin.feathr.config.join.Date _dateMember = null; + private com.linkedin.feathr.config.join.HourTime _hourTimeMember = null; + private AbsoluteTimeRange.StartTime.ChangeListener __changeListener = new AbsoluteTimeRange.StartTime.ChangeListener(this); + private final static DataSchema MEMBER_Date = SCHEMA.getTypeByMemberKey("date"); + private final static DataSchema MEMBER_HourTime = SCHEMA.getTypeByMemberKey("hourTime"); + + public StartTime() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public StartTime(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static AbsoluteTimeRange.StartTime createWithDate(com.linkedin.feathr.config.join.Date value) { + AbsoluteTimeRange.StartTime newUnion = new AbsoluteTimeRange.StartTime(); + newUnion.setDate(value); + return newUnion; + } + + public boolean isDate() { + return memberIs("date"); + } + + public com.linkedin.feathr.config.join.Date getDate() { + checkNotNull(); + if (_dateMember!= null) { + return _dateMember; + } + Object __rawValue = super._map.get("date"); + _dateMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.Date(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _dateMember; + } + + public void setDate(com.linkedin.feathr.config.join.Date value) { + checkNotNull(); + super._map.clear(); + _dateMember = value; + CheckedUtil.putWithoutChecking(super._map, "date", value.data()); + } + + public static AbsoluteTimeRange.StartTime createWithHourTime(com.linkedin.feathr.config.join.HourTime value) { + AbsoluteTimeRange.StartTime newUnion = new AbsoluteTimeRange.StartTime(); + newUnion.setHourTime(value); + return newUnion; + } + + public boolean isHourTime() { + return memberIs("hourTime"); + } + + public com.linkedin.feathr.config.join.HourTime getHourTime() { + checkNotNull(); + if (_hourTimeMember!= null) { + return _hourTimeMember; + } + Object __rawValue = super._map.get("hourTime"); + _hourTimeMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.HourTime(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _hourTimeMember; + } + + public void setHourTime(com.linkedin.feathr.config.join.HourTime value) { + checkNotNull(); + super._map.clear(); + _hourTimeMember = value; + CheckedUtil.putWithoutChecking(super._map, "hourTime", value.data()); + } + + public static AbsoluteTimeRange.StartTime.ProjectionMask createMask() { + return new AbsoluteTimeRange.StartTime.ProjectionMask(); + } + + @Override + public AbsoluteTimeRange.StartTime clone() + throws CloneNotSupportedException + { + AbsoluteTimeRange.StartTime __clone = ((AbsoluteTimeRange.StartTime) super.clone()); + __clone.__changeListener = new AbsoluteTimeRange.StartTime.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public AbsoluteTimeRange.StartTime copy() + throws CloneNotSupportedException + { + AbsoluteTimeRange.StartTime __copy = ((AbsoluteTimeRange.StartTime) super.copy()); + __copy._dateMember = null; + __copy._hourTimeMember = null; + __copy.__changeListener = new AbsoluteTimeRange.StartTime.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final AbsoluteTimeRange.StartTime __objectRef; + + private ChangeListener(AbsoluteTimeRange.StartTime reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "date": + __objectRef._dateMember = null; + break; + case "hourTime": + __objectRef._hourTimeMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.config.join.Date.Fields Date() { + return new com.linkedin.feathr.config.join.Date.Fields(getPathComponents(), "date"); + } + + public com.linkedin.feathr.config.join.HourTime.Fields HourTime() { + return new com.linkedin.feathr.config.join.HourTime.Fields(getPathComponents(), "hourTime"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.Date.ProjectionMask _DateMask; + private com.linkedin.feathr.config.join.HourTime.ProjectionMask _HourTimeMask; + + ProjectionMask() { + super(3); + } + + public AbsoluteTimeRange.StartTime.ProjectionMask withDate(Function nestedMask) { + _DateMask = nestedMask.apply(((_DateMask == null)?com.linkedin.feathr.config.join.Date.createMask():_DateMask)); + getDataMap().put("date", _DateMask.getDataMap()); + return this; + } + + public AbsoluteTimeRange.StartTime.ProjectionMask withHourTime(Function nestedMask) { + _HourTimeMask = nestedMask.apply(((_HourTimeMask == null)?com.linkedin.feathr.config.join.HourTime.createMask():_HourTimeMask)); + getDataMap().put("hourTime", _HourTimeMask.getDataMap()); + return this; + } + + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Date.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Date.java new file mode 100644 index 000000000..c27cfb272 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Date.java @@ -0,0 +1,575 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * Represents a date in a calendar year including day, year and month + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\Date.pdl.") +public class Date + extends RecordTemplate +{ + + private final static Date.Fields _fields = new Date.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}", SchemaFormatType.PDL)); + private Integer _dayField = null; + private Integer _monthField = null; + private Integer _yearField = null; + private Date.ChangeListener __changeListener = new Date.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Day = SCHEMA.getField("day"); + private final static RecordDataSchema.Field FIELD_Month = SCHEMA.getField("month"); + private final static RecordDataSchema.Field FIELD_Year = SCHEMA.getField("year"); + + public Date() { + super(new DataMap(4, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public Date(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static Date.Fields fields() { + return _fields; + } + + public static Date.ProjectionMask createMask() { + return new Date.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for day + * + * @see Date.Fields#day + */ + public boolean hasDay() { + if (_dayField!= null) { + return true; + } + return super._map.containsKey("day"); + } + + /** + * Remover for day + * + * @see Date.Fields#day + */ + public void removeDay() { + super._map.remove("day"); + } + + /** + * Getter for day + * + * @see Date.Fields#day + */ + public Integer getDay(GetMode mode) { + switch (mode) { + case STRICT: + return getDay(); + case DEFAULT: + case NULL: + if (_dayField!= null) { + return _dayField; + } else { + Object __rawValue = super._map.get("day"); + _dayField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _dayField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for day + * + * @return + * Required field. Could be null for partial record. + * @see Date.Fields#day + */ + @Nonnull + public Integer getDay() { + if (_dayField!= null) { + return _dayField; + } else { + Object __rawValue = super._map.get("day"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("day"); + } + _dayField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _dayField; + } + } + + /** + * Setter for day + * + * @see Date.Fields#day + */ + public Date setDay(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setDay(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field day of com.linkedin.feathr.config.join.Date"); + } else { + CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); + _dayField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeDay(); + } else { + CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); + _dayField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); + _dayField = value; + } + break; + } + return this; + } + + /** + * Setter for day + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Date.Fields#day + */ + public Date setDay( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field day of com.linkedin.feathr.config.join.Date to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); + _dayField = value; + } + return this; + } + + /** + * Setter for day + * + * @see Date.Fields#day + */ + public Date setDay(int value) { + CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); + _dayField = value; + return this; + } + + /** + * Existence checker for month + * + * @see Date.Fields#month + */ + public boolean hasMonth() { + if (_monthField!= null) { + return true; + } + return super._map.containsKey("month"); + } + + /** + * Remover for month + * + * @see Date.Fields#month + */ + public void removeMonth() { + super._map.remove("month"); + } + + /** + * Getter for month + * + * @see Date.Fields#month + */ + public Integer getMonth(GetMode mode) { + switch (mode) { + case STRICT: + return getMonth(); + case DEFAULT: + case NULL: + if (_monthField!= null) { + return _monthField; + } else { + Object __rawValue = super._map.get("month"); + _monthField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _monthField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for month + * + * @return + * Required field. Could be null for partial record. + * @see Date.Fields#month + */ + @Nonnull + public Integer getMonth() { + if (_monthField!= null) { + return _monthField; + } else { + Object __rawValue = super._map.get("month"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("month"); + } + _monthField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _monthField; + } + } + + /** + * Setter for month + * + * @see Date.Fields#month + */ + public Date setMonth(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setMonth(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field month of com.linkedin.feathr.config.join.Date"); + } else { + CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); + _monthField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeMonth(); + } else { + CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); + _monthField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); + _monthField = value; + } + break; + } + return this; + } + + /** + * Setter for month + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Date.Fields#month + */ + public Date setMonth( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field month of com.linkedin.feathr.config.join.Date to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); + _monthField = value; + } + return this; + } + + /** + * Setter for month + * + * @see Date.Fields#month + */ + public Date setMonth(int value) { + CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); + _monthField = value; + return this; + } + + /** + * Existence checker for year + * + * @see Date.Fields#year + */ + public boolean hasYear() { + if (_yearField!= null) { + return true; + } + return super._map.containsKey("year"); + } + + /** + * Remover for year + * + * @see Date.Fields#year + */ + public void removeYear() { + super._map.remove("year"); + } + + /** + * Getter for year + * + * @see Date.Fields#year + */ + public Integer getYear(GetMode mode) { + switch (mode) { + case STRICT: + return getYear(); + case DEFAULT: + case NULL: + if (_yearField!= null) { + return _yearField; + } else { + Object __rawValue = super._map.get("year"); + _yearField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _yearField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for year + * + * @return + * Required field. Could be null for partial record. + * @see Date.Fields#year + */ + @Nonnull + public Integer getYear() { + if (_yearField!= null) { + return _yearField; + } else { + Object __rawValue = super._map.get("year"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("year"); + } + _yearField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _yearField; + } + } + + /** + * Setter for year + * + * @see Date.Fields#year + */ + public Date setYear(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setYear(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field year of com.linkedin.feathr.config.join.Date"); + } else { + CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); + _yearField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeYear(); + } else { + CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); + _yearField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); + _yearField = value; + } + break; + } + return this; + } + + /** + * Setter for year + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Date.Fields#year + */ + public Date setYear( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field year of com.linkedin.feathr.config.join.Date to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); + _yearField = value; + } + return this; + } + + /** + * Setter for year + * + * @see Date.Fields#year + */ + public Date setYear(int value) { + CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); + _yearField = value; + return this; + } + + @Override + public Date clone() + throws CloneNotSupportedException + { + Date __clone = ((Date) super.clone()); + __clone.__changeListener = new Date.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public Date copy() + throws CloneNotSupportedException + { + Date __copy = ((Date) super.copy()); + __copy._monthField = null; + __copy._yearField = null; + __copy._dayField = null; + __copy.__changeListener = new Date.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final Date __objectRef; + + private ChangeListener(Date reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "month": + __objectRef._monthField = null; + break; + case "year": + __objectRef._yearField = null; + break; + case "day": + __objectRef._dayField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * day + * + */ + public PathSpec day() { + return new PathSpec(getPathComponents(), "day"); + } + + /** + * month + * + */ + public PathSpec month() { + return new PathSpec(getPathComponents(), "month"); + } + + /** + * year + * + */ + public PathSpec year() { + return new PathSpec(getPathComponents(), "year"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(4); + } + + /** + * day + * + */ + public Date.ProjectionMask withDay() { + getDataMap().put("day", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * month + * + */ + public Date.ProjectionMask withMonth() { + getDataMap().put("month", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * year + * + */ + public Date.ProjectionMask withYear() { + getDataMap().put("year", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/FrameFeatureJoinConfig.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/FrameFeatureJoinConfig.java new file mode 100644 index 000000000..6400da9df --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/FrameFeatureJoinConfig.java @@ -0,0 +1,520 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * The join config consists of 2 parts, settings and features section. + * Settings is related to the general settings corresponding to joining the input data set with the + * features, currently there are time related settings, but this can be extended to other settings as well. + * Features to be joined are described by list of Keys and featureName and featureAlias. + * Features in the feature list should be joined to the user's input data. + * matching the key in the input data. + * For example, + * key is ["key1"] and join feature1 and feature2 with input data + * settings: { // optional field + * inputDataTimeSettings: { + * absoluteTimeRange: { + * startTime: Date(year=2020, month=4, day=28) + * endTime: Date(year=2020, month=5, day=5) + * } + * } + * joinTimeSettings: { + * timestampColumn: { + * def: timestamp + * format: yyyy-MM-dd + * } + * simulateTimeDelay: 5d + * } + * } + * features=[ + * JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature1" + * AbsoluteDateRange(startDate: Date(year=2020, month=5, day=1), + * endTime: Date(year=2020, month=5, day=5)) + * }, JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature2" + * overrideTimeDelay: 5d + * }, JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature3" + * RelativeDateRange(numDays: 5, + * offset: 3) + * }, JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature4" + * } + * ] + * + * Here, the keys are corresponding to column names in the input FeaturizedDataset, which will be used + * to join the feature source. Feature name is canonical feathr feature names. + * Each feature can also have a set of optional time-related parameters. These parameter override the ones provided in + * the settings section and are applicable only to the particular feature. + * Feature join config operation. + * + * All these PDLs are moved to feathr MP:- https://rb.corp.linkedin.com/r/2356512/ + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\FrameFeatureJoinConfig.pdl.") +public class FrameFeatureJoinConfig + extends RecordTemplate +{ + + private final static FrameFeatureJoinConfig.Fields _fields = new FrameFeatureJoinConfig.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The join config consists of 2 parts, settings and features section.\r\nSettings is related to the general settings corresponding to joining the input data set with the\r\nfeatures, currently there are time related settings, but this can be extended to other settings as well.\r\nFeatures to be joined are described by list of Keys and featureName and featureAlias.\r\nFeatures in the feature list should be joined to the user's input data.\r\nmatching the key in the input data.\r\nFor example,\r\nkey is [\"key1\"] and join feature1 and feature2 with input data\r\n settings: { // optional field\r\n inputDataTimeSettings: {\r\n absoluteTimeRange: {\r\n startTime: Date(year=2020, month=4, day=28)\r\n endTime: Date(year=2020, month=5, day=5)\r\n }\r\n }\r\n joinTimeSettings: {\r\n timestampColumn: {\r\n def: timestamp\r\n format: yyyy-MM-dd\r\n }\r\n simulateTimeDelay: 5d\r\n }\r\n }\r\n features=[\r\n JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature1\"\r\n AbsoluteDateRange(startDate: Date(year=2020, month=5, day=1),\r\n endTime: Date(year=2020, month=5, day=5))\r\n }, JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature2\"\r\n overrideTimeDelay: 5d\r\n }, JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature3\"\r\n RelativeDateRange(numDays: 5,\r\n offset: 3)\r\n }, JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature4\"\r\n }\r\n ]\r\n\r\nHere, the keys are corresponding to column names in the input FeaturizedDataset, which will be used\r\nto join the feature source. Feature name is canonical feathr feature names.\r\nEach feature can also have a set of optional time-related parameters. These parameter override the ones provided in\r\nthe settings section and are applicable only to the particular feature.\r\nFeature join config operation.\r\n\r\nAll these PDLs are moved to feathr MP:- https://rb.corp.linkedin.com/r/2356512/*/record FrameFeatureJoinConfig{/**settings required for joining input featurized dataset with the feature data.*/settings:optional/**The settings section contains all the config parameters required for the joining of input dataset with the\r\nfeature data. As of now, we have only time related parameters, but in future this can be expanded.\r\nThis section has configs related to:-\r\na. How do I load the input dataset if it is time sensitive?\r\nb. How do I specify the join parameters for input dataset?\r\nFor more details - https://docs.google.com/document/d/1C6u2CKWSmOmHDQEL8Ovm5V5ZZFKhC_HdxVxU9D1F9lg/edit#\r\nsettings: {\r\n inputDataTimeSettings: {\r\n absoluteTimeRange: {\r\n startTime: 20200809\r\n endTime: 20200810\r\n timeFormat: yyyyMMdd\r\n }\r\n }\r\n joinTimeSettings: {\r\n useLatestFeatureData: true\r\n }\r\n}*/record Settings{/**Config parameters related to loading of the time sensitive input data. Contains parameters related to restricting the\r\nsize of the input data with respect to the timestamp column.*/inputDataTimeSettings:optional/**The data time settings pertaining to how much of the input dataset is to be loaded from the timestamp column. This is a way in which\r\nthe input data can be restricted to allow only a fixed interval of dates to be joined with the feature data. This restriction\r\nwill apply on the timestamp column of the input data.\r\ninputDataTimeSettings: {\r\n absoluteTimeRange: {\r\n startTime: Date(year=2020, month=8, day=8)\r\n endTime: Date(year=2020, month=8, day=10)\r\n }\r\n (or)\r\n relativeTimeRange: {\r\n offset: TimeOffset(length=1, unit=\"DAY\")\r\n window: TimeWindow(length=1, unit=\"DAY\")\r\n }\r\n}*/record InputDataTimeSettings{/**Union of [[AbsoluteTimeRange]] and [[RelativeTimeRange]].\r\nIt indicates the range of input data which is to be loaded. This field generally refers to how much of the input\r\ndata should be restricted using the time in the timestamp column.\r\n\r\nFor example,\r\na. startDate: \"20200522\", endDate: \"20200525\" implies this feature should be joined with the input data starting from\r\n22nd May 2020 to 25th May, 2020 with both dates included.\r\nWe only support yyyyMMdd format for this. In future, if there is a request, we can\r\nadd support for other date time formats as well.\r\n\r\nb. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020\r\ntill 11/04/2020 willl be joined.*/timeRange:union[absoluteTimeRange:/**The absolute time range with start and end time being required fields.\r\nIt accepts a start time and an end time which should be specifiied using the [[Date.pdl]] or the [[HourTime.pdl]] class.\r\nThis model can be used to represent time range in daily or hourly interval.\r\nabsoluteTimeRange: {\r\n startTime: TimeHour(day=1, month=1, year=2020, hour=13)\r\n endTime: TimeHour(day=3, month=1, year=2020, hour=2)\r\n }\r\n(or)\r\nabsoluteTimeRange: {\r\n startTime: Date(day=1, month=1, year=2020)\r\n endTime: Date(day=3, month=1, year=2020)\r\n }\r\nendTime and startTime should always have the same granularity, ie - Daily or Hourly.\r\nendTme > startTime*/record AbsoluteTimeRange{/**start time of the date range, in daily or hourly format with the start date included in the range.*/startTime:union[date:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}hourTime:/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}]/**end date of the date range, in daily or hourly format with the end date included in the range.*/endTime:union[date:Date,hourTime:HourTime]}relativeTimeRange:/**The time range represented relative to the current timestamp. It uses the current system time as the reference and can be used to\r\nexpress a range of times with respect to the current time.\r\nExample, - If current time is 01/01/2020, window is 3 days, and offset is 1 day (unit can be day or hour).\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"DAY\"), offset=TimeOffset(length=1, unit=\"Day\"))\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"HOUR\"))*/record RelativeTimeRange{/**Window is the number of time units from the reference time units to look back to obtain the timeRange.\r\nFor example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020\r\ntill 11/05/2020 (both days included).\r\nwindow >= 1 TimeUnit*/window:/**Represents a length of time along with the corresponding time unit (DAY, HOUR).*/record TimeWindow{/**Amount of the duration in TimeUnits. Can be greater or equal to 1.*/@validate.positive,length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}/**Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time.\r\nFor example, if dateOffset is 4, and window is 2 days, then reference time\r\nwill be 4 days ago from today.\r\nExample - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020.\r\nThis will always take the window's timeUnits.*/@validate.integerRange.min=0,offset:long=0}]}/**This contains all the parameters required to join the time sensitive input data with the feature data.*/joinTimeSettings:optional/**JoinTimeSettings contains all the parameters required to join the time sensitive input data with the feature data.\r\nThe input data can be time sensitive in two ways:-\r\na. Have a timestamp column\r\nb. Always join with the latest available feature data. In this case, we do not require a timestamp column.\r\nc. The file path is time-partition and the path time is used for the join\r\n(Todo - Add useTimePartitionPattern field in this section)\r\nIn this section, the user needs to let feathr know which of the above properties is to be used for the join.*/typeref JoinTimeSettings=union[useLatestJoinTimeSettings:/**Settings needed when the input data is to be joined with the latest available feature data.\r\njoinTimeSettings: {\r\n useLatestFeatureData: true\r\n}*/record UseLatestJoinTimeSettings{/**Boolean value, if set to true, indicates that the latest available feature data is to be used for joining.\r\nWhen useLatestFeatureData is set, there should be no other time-based parameters.*/useLatestFeatureData:boolean=true}timestampColJoinTimeSettings:/**Settings needed when the input data has a timestamp which should be used for the join.\r\njoinTimeSettings: {\r\n timestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }\r\n simulateTimeDelay: 1d\r\n}*/record TimestampColJoinTimeSettings{/**The timestamp column name and timeformat which should be used for joining with the feature data.\r\nRefer to [[TimestampColumn]].\r\nExample, TimestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }*/timestampColumn:/**Timestamp column of the input featureiized dataset, which is to be used for the join.\r\ntimestampColumn: {\r\n def: timestamp\r\n format: yyyyMMdd\r\n }*/record TimestampColumn{/**The definiton of the timestamp column, which can be a sql expression involving the timestamp column\r\nor just the column name\r\nExample:- definition: timestamp, timestamp + 10000000.*/definition:union[columnName:string,sparkSqlExpression:/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}]/**Format of the timestamp column. Must confer to java's timestampFormatter or can be\r\nepoch or epoch_millis.\r\nExample:- epoch, epoch_millis, yyyy/MM/dd*/format:/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string}/**An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted\r\nfrom the input data timestamp while joining with the feature data.\r\nWe do support negative time delays.*/simulateTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:TimeUnit}}]}/**Array of joining features.\r\n\r\nValidation rules:\r\n- The array must be non-empty.*/features:array[/**JoiningFeature is the feature section of the join config. This section consists of information pertaining to a feature\r\nwhich is to be joined:-\r\na. The join keys of the input data, with which this feature is to be joined.\r\nb. name of the feature\r\nc. optional timeRange of the input data which is to be joined with this feature.\r\nd. optional overrideTimeDelay if this feature needs a different simulate time delay other than the one mentioned.\r\n\r\nThis is a required section of the the join config.\r\nExample,\r\n a. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature1\"\r\n AbsoluteDateRange(startDate: Date(year=2020, month=5, day=5),\r\n endDate: Date(year=2020, month=5, day=7))\r\n }\r\n b. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature2\"\r\n overrideTimeDelay: TimeDelay(length=1, unit=\"DAY\")\r\n }\r\n c. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature3\"\r\n RelativeDateRange(numDays: 5,\r\n offset: 3)\r\n }*/record JoiningFeature{/**Keys to join input with feature source, the field name of the key in the input featuized dataset.*/keys:array[string]/**Feature name as defined in feathr's feature definition configuration.\r\n\r\nCurrently the column in the output FDS that holds this feature will have the same name as feature name.\r\nIf multiple joined features have the same name and no alias is defined for them, feathr will prepend the keys to the feature name.\r\n\r\nIn the future, if \"featureAlias\" is not set, the column in the output FDS that holds this feature will have the same name as feature name.\r\nIf multiple joined features have the same name and no alias is defined for them, the join operation will fail\r\n(to avoid produciing two columns in the output FDS with the same name).*/frameFeatureName:string/**The development of this is in progress. This is not in use for now.\r\n\r\nThe name to be used for the column in the output FDS that contains the values from this joined feature.\r\nIf not set, the name of the feature (frameFeatureName) will be used for the output column.\r\nFor example, if the user request joining a feature named \"careers_job_listTime\" and provides no alias,\r\nthe output FDS will contain a column called \"careers_job_listTime\". However, if the user sets \"featureAlias\" to \"list_time\",\r\nthe column will be named \"list_time\".\r\n\r\nfeature alias can be useful for in a few cases:\r\n - If the user prefers to use a name different than the feathr name in their model,\r\nthey can use an alias to control the name of the column in the output FDS.\r\n - Sometimes, the training datas needs to have two features that are from the same feathr feature.\r\nFor example, if we are modeing the problem of the probability of a member A (viewer) seeing the profile of member B\r\n(viewee) and we want to use the skills of both viewer and viewee as features, we need to join feathr feature\r\n\"member_skills\" of member A with feathr feature \"member_skills\" of member B. That is, the two features are the same\r\nfeature but for different entiity ids). The default behavior of join is to name the output column name using the feathr\r\nfeature name, but in a case like the above case, that would result in two columns with the same name,\r\nwhich is not valid for FDS. In these cases, the user has to provide an alias for at least one of these joined features.\r\nFor example, the user can use featureAliases such as \"viewer_skills\" and \"viewee_skills\".\r\nIn these cases, featureAliases becomes mandatory.*/featureAlias:optional string/**dateRange is used in Time-based joins, which refers to the situation when one or multiple days of input data needs\r\nto be used for training.\r\nOne of the common use cases where this is used, is in training with some time-insensitive features, or\r\ntraining pipeline that always use the full day data, one day before running (since there is only partial data for today).\r\nThe time for the input featurized dataset can be set using this field.\r\nHourly data is not allowed in this case.\r\n\r\nFor example,\r\na. startDate: \"20200522\", endDate: \"20200525\" implies this feature should be joined with the input data starting from\r\n22nd May 2020 to 25th May, 2020 with both dates included.\r\nWe only support yyyyMMdd format for this. In future, if there is a request, we can\r\nadd support for other date time formats as well.\r\n\r\nb. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020\r\ntill 11/04/2020 willl be joined.\r\n\r\nP.S - This is different from the timeRange used in settings as the settings startTime is applicable for the entire input data,\r\nwhile this a feature level setting. Also, we do not support hourly time here.*/dateRange:optional union[absoluteDateRange:/**The absolute date range with start and end date being required fields.\r\nIt accepts a start date and an end date which should be specifiied using the [[Date.pdl]] class.\r\nabsoluteDateRange: {\r\n startDate: Date(day=1, month=1, year=2020)\r\n endDate: Date(day=3, month=1, year=2020)\r\n }\r\n In this case, the endDate > startDate.*/record AbsoluteDateRange{/**start date of the date range, with the start date included in the range.*/startDate:Date/**end date of the date range, with the end date included in the range.*/endDate:Date}relativeDateRange:/**The date range represented relative to the current date. It uses the current system date as the reference and can be used to\r\nexpress a range of dates with respect to the current date.\r\nExample, - If current date is 01/01/2020, window is 3, and offset 1 (unit is number of days)\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nIf dateOffset is not specified, it defaults to 0.\r\nrelativeDateRange: RelativeDateRange(numDays=2, dateOffset=1)\r\nrelativeDateRange: RelativeDateRange(numDays=5)*/record RelativeDateRange{/**Represents a length of time.\r\nnumDays is the window from the reference date to look back to obtain a dateRange.\r\nFor example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020\r\ntill 11/05/2020.*/@validate.positive={}numDays:long/**Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date\r\nwill be 4 days ago from today.*/dateOffset:long=0}]/**The override time delay parameter which will override the global simulate time delay specified in the settings section for\r\nthe particular feature.\r\nThis parameter is only applicable when the simulate time delay is set in the settings section\r\nFor example, let us say the global simulate delay was 5d, and the overrideTimeDelay is set to 3d.\r\nThen, for this specificc feature, a simulate delay of 3d will be applied.*/overrideTimeDelay:optional TimeOffset}]}", SchemaFormatType.PDL)); + private Settings _settingsField = null; + private JoiningFeatureArray _featuresField = null; + private FrameFeatureJoinConfig.ChangeListener __changeListener = new FrameFeatureJoinConfig.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Settings = SCHEMA.getField("settings"); + private final static RecordDataSchema.Field FIELD_Features = SCHEMA.getField("features"); + + public FrameFeatureJoinConfig() { + super(new DataMap(3, 0.75F), SCHEMA, 3); + addChangeListener(__changeListener); + } + + public FrameFeatureJoinConfig(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static FrameFeatureJoinConfig.Fields fields() { + return _fields; + } + + public static FrameFeatureJoinConfig.ProjectionMask createMask() { + return new FrameFeatureJoinConfig.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for settings + * + * @see FrameFeatureJoinConfig.Fields#settings + */ + public boolean hasSettings() { + if (_settingsField!= null) { + return true; + } + return super._map.containsKey("settings"); + } + + /** + * Remover for settings + * + * @see FrameFeatureJoinConfig.Fields#settings + */ + public void removeSettings() { + super._map.remove("settings"); + } + + /** + * Getter for settings + * + * @see FrameFeatureJoinConfig.Fields#settings + */ + public Settings getSettings(GetMode mode) { + return getSettings(); + } + + /** + * Getter for settings + * + * @return + * Optional field. Always check for null. + * @see FrameFeatureJoinConfig.Fields#settings + */ + @Nullable + public Settings getSettings() { + if (_settingsField!= null) { + return _settingsField; + } else { + Object __rawValue = super._map.get("settings"); + _settingsField = ((__rawValue == null)?null:new Settings(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _settingsField; + } + } + + /** + * Setter for settings + * + * @see FrameFeatureJoinConfig.Fields#settings + */ + public FrameFeatureJoinConfig setSettings(Settings value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setSettings(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeSettings(); + } else { + CheckedUtil.putWithoutChecking(super._map, "settings", value.data()); + _settingsField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "settings", value.data()); + _settingsField = value; + } + break; + } + return this; + } + + /** + * Setter for settings + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see FrameFeatureJoinConfig.Fields#settings + */ + public FrameFeatureJoinConfig setSettings( + @Nonnull + Settings value) { + if (value == null) { + throw new NullPointerException("Cannot set field settings of com.linkedin.feathr.config.join.FrameFeatureJoinConfig to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "settings", value.data()); + _settingsField = value; + } + return this; + } + + /** + * Existence checker for features + * + * @see FrameFeatureJoinConfig.Fields#features + */ + public boolean hasFeatures() { + if (_featuresField!= null) { + return true; + } + return super._map.containsKey("features"); + } + + /** + * Remover for features + * + * @see FrameFeatureJoinConfig.Fields#features + */ + public void removeFeatures() { + super._map.remove("features"); + } + + /** + * Getter for features + * + * @see FrameFeatureJoinConfig.Fields#features + */ + public JoiningFeatureArray getFeatures(GetMode mode) { + switch (mode) { + case STRICT: + return getFeatures(); + case DEFAULT: + case NULL: + if (_featuresField!= null) { + return _featuresField; + } else { + Object __rawValue = super._map.get("features"); + _featuresField = ((__rawValue == null)?null:new JoiningFeatureArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _featuresField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for features + * + * @return + * Required field. Could be null for partial record. + * @see FrameFeatureJoinConfig.Fields#features + */ + @Nonnull + public JoiningFeatureArray getFeatures() { + if (_featuresField!= null) { + return _featuresField; + } else { + Object __rawValue = super._map.get("features"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("features"); + } + _featuresField = ((__rawValue == null)?null:new JoiningFeatureArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _featuresField; + } + } + + /** + * Setter for features + * + * @see FrameFeatureJoinConfig.Fields#features + */ + public FrameFeatureJoinConfig setFeatures(JoiningFeatureArray value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFeatures(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field features of com.linkedin.feathr.config.join.FrameFeatureJoinConfig"); + } else { + CheckedUtil.putWithoutChecking(super._map, "features", value.data()); + _featuresField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeFeatures(); + } else { + CheckedUtil.putWithoutChecking(super._map, "features", value.data()); + _featuresField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "features", value.data()); + _featuresField = value; + } + break; + } + return this; + } + + /** + * Setter for features + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see FrameFeatureJoinConfig.Fields#features + */ + public FrameFeatureJoinConfig setFeatures( + @Nonnull + JoiningFeatureArray value) { + if (value == null) { + throw new NullPointerException("Cannot set field features of com.linkedin.feathr.config.join.FrameFeatureJoinConfig to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "features", value.data()); + _featuresField = value; + } + return this; + } + + @Override + public FrameFeatureJoinConfig clone() + throws CloneNotSupportedException + { + FrameFeatureJoinConfig __clone = ((FrameFeatureJoinConfig) super.clone()); + __clone.__changeListener = new FrameFeatureJoinConfig.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public FrameFeatureJoinConfig copy() + throws CloneNotSupportedException + { + FrameFeatureJoinConfig __copy = ((FrameFeatureJoinConfig) super.copy()); + __copy._settingsField = null; + __copy._featuresField = null; + __copy.__changeListener = new FrameFeatureJoinConfig.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final FrameFeatureJoinConfig __objectRef; + + private ChangeListener(FrameFeatureJoinConfig reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "settings": + __objectRef._settingsField = null; + break; + case "features": + __objectRef._featuresField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * settings required for joining input featurized dataset with the feature data. + * + */ + public com.linkedin.feathr.config.join.Settings.Fields settings() { + return new com.linkedin.feathr.config.join.Settings.Fields(getPathComponents(), "settings"); + } + + /** + * Array of joining features. + * + * Validation rules: + * - The array must be non-empty. + * + */ + public com.linkedin.feathr.config.join.JoiningFeatureArray.Fields features() { + return new com.linkedin.feathr.config.join.JoiningFeatureArray.Fields(getPathComponents(), "features"); + } + + /** + * Array of joining features. + * + * Validation rules: + * - The array must be non-empty. + * + */ + public PathSpec features(Integer start, Integer count) { + PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "features"); + if (start!= null) { + arrayPathSpec.setAttribute("start", start); + } + if (count!= null) { + arrayPathSpec.setAttribute("count", count); + } + return arrayPathSpec; + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.Settings.ProjectionMask _settingsMask; + private com.linkedin.feathr.config.join.JoiningFeatureArray.ProjectionMask _featuresMask; + + ProjectionMask() { + super(3); + } + + /** + * settings required for joining input featurized dataset with the feature data. + * + */ + public FrameFeatureJoinConfig.ProjectionMask withSettings(Function nestedMask) { + _settingsMask = nestedMask.apply(((_settingsMask == null)?Settings.createMask():_settingsMask)); + getDataMap().put("settings", _settingsMask.getDataMap()); + return this; + } + + /** + * settings required for joining input featurized dataset with the feature data. + * + */ + public FrameFeatureJoinConfig.ProjectionMask withSettings() { + _settingsMask = null; + getDataMap().put("settings", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Array of joining features. + * + * Validation rules: + * - The array must be non-empty. + * + */ + public FrameFeatureJoinConfig.ProjectionMask withFeatures(Function nestedMask) { + _featuresMask = nestedMask.apply(((_featuresMask == null)?JoiningFeatureArray.createMask():_featuresMask)); + getDataMap().put("features", _featuresMask.getDataMap()); + return this; + } + + /** + * Array of joining features. + * + * Validation rules: + * - The array must be non-empty. + * + */ + public FrameFeatureJoinConfig.ProjectionMask withFeatures() { + _featuresMask = null; + getDataMap().put("features", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Array of joining features. + * + * Validation rules: + * - The array must be non-empty. + * + */ + public FrameFeatureJoinConfig.ProjectionMask withFeatures(Function nestedMask, Integer start, Integer count) { + _featuresMask = nestedMask.apply(((_featuresMask == null)?JoiningFeatureArray.createMask():_featuresMask)); + getDataMap().put("features", _featuresMask.getDataMap()); + if (start!= null) { + getDataMap().getDataMap("features").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("features").put("$count", count); + } + return this; + } + + /** + * Array of joining features. + * + * Validation rules: + * - The array must be non-empty. + * + */ + public FrameFeatureJoinConfig.ProjectionMask withFeatures(Integer start, Integer count) { + _featuresMask = null; + getDataMap().put("features", new DataMap(3)); + if (start!= null) { + getDataMap().getDataMap("features").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("features").put("$count", count); + } + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/HourTime.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/HourTime.java new file mode 100644 index 000000000..7421350ae --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/HourTime.java @@ -0,0 +1,727 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * Time with hourly granularity + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\HourTime.pdl.") +public class HourTime + extends RecordTemplate +{ + + private final static HourTime.Fields _fields = new HourTime.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}", SchemaFormatType.PDL)); + private Integer _dayField = null; + private Integer _monthField = null; + private Integer _yearField = null; + private Integer _hourField = null; + private HourTime.ChangeListener __changeListener = new HourTime.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Day = SCHEMA.getField("day"); + private final static RecordDataSchema.Field FIELD_Month = SCHEMA.getField("month"); + private final static RecordDataSchema.Field FIELD_Year = SCHEMA.getField("year"); + private final static RecordDataSchema.Field FIELD_Hour = SCHEMA.getField("hour"); + + public HourTime() { + super(new DataMap(6, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public HourTime(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static HourTime.Fields fields() { + return _fields; + } + + public static HourTime.ProjectionMask createMask() { + return new HourTime.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for day + * + * @see HourTime.Fields#day + */ + public boolean hasDay() { + if (_dayField!= null) { + return true; + } + return super._map.containsKey("day"); + } + + /** + * Remover for day + * + * @see HourTime.Fields#day + */ + public void removeDay() { + super._map.remove("day"); + } + + /** + * Getter for day + * + * @see HourTime.Fields#day + */ + public Integer getDay(GetMode mode) { + switch (mode) { + case STRICT: + return getDay(); + case DEFAULT: + case NULL: + if (_dayField!= null) { + return _dayField; + } else { + Object __rawValue = super._map.get("day"); + _dayField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _dayField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for day + * + * @return + * Required field. Could be null for partial record. + * @see HourTime.Fields#day + */ + @Nonnull + public Integer getDay() { + if (_dayField!= null) { + return _dayField; + } else { + Object __rawValue = super._map.get("day"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("day"); + } + _dayField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _dayField; + } + } + + /** + * Setter for day + * + * @see HourTime.Fields#day + */ + public HourTime setDay(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setDay(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field day of com.linkedin.feathr.config.join.HourTime"); + } else { + CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); + _dayField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeDay(); + } else { + CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); + _dayField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); + _dayField = value; + } + break; + } + return this; + } + + /** + * Setter for day + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see HourTime.Fields#day + */ + public HourTime setDay( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field day of com.linkedin.feathr.config.join.HourTime to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); + _dayField = value; + } + return this; + } + + /** + * Setter for day + * + * @see HourTime.Fields#day + */ + public HourTime setDay(int value) { + CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); + _dayField = value; + return this; + } + + /** + * Existence checker for month + * + * @see HourTime.Fields#month + */ + public boolean hasMonth() { + if (_monthField!= null) { + return true; + } + return super._map.containsKey("month"); + } + + /** + * Remover for month + * + * @see HourTime.Fields#month + */ + public void removeMonth() { + super._map.remove("month"); + } + + /** + * Getter for month + * + * @see HourTime.Fields#month + */ + public Integer getMonth(GetMode mode) { + switch (mode) { + case STRICT: + return getMonth(); + case DEFAULT: + case NULL: + if (_monthField!= null) { + return _monthField; + } else { + Object __rawValue = super._map.get("month"); + _monthField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _monthField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for month + * + * @return + * Required field. Could be null for partial record. + * @see HourTime.Fields#month + */ + @Nonnull + public Integer getMonth() { + if (_monthField!= null) { + return _monthField; + } else { + Object __rawValue = super._map.get("month"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("month"); + } + _monthField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _monthField; + } + } + + /** + * Setter for month + * + * @see HourTime.Fields#month + */ + public HourTime setMonth(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setMonth(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field month of com.linkedin.feathr.config.join.HourTime"); + } else { + CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); + _monthField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeMonth(); + } else { + CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); + _monthField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); + _monthField = value; + } + break; + } + return this; + } + + /** + * Setter for month + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see HourTime.Fields#month + */ + public HourTime setMonth( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field month of com.linkedin.feathr.config.join.HourTime to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); + _monthField = value; + } + return this; + } + + /** + * Setter for month + * + * @see HourTime.Fields#month + */ + public HourTime setMonth(int value) { + CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); + _monthField = value; + return this; + } + + /** + * Existence checker for year + * + * @see HourTime.Fields#year + */ + public boolean hasYear() { + if (_yearField!= null) { + return true; + } + return super._map.containsKey("year"); + } + + /** + * Remover for year + * + * @see HourTime.Fields#year + */ + public void removeYear() { + super._map.remove("year"); + } + + /** + * Getter for year + * + * @see HourTime.Fields#year + */ + public Integer getYear(GetMode mode) { + switch (mode) { + case STRICT: + return getYear(); + case DEFAULT: + case NULL: + if (_yearField!= null) { + return _yearField; + } else { + Object __rawValue = super._map.get("year"); + _yearField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _yearField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for year + * + * @return + * Required field. Could be null for partial record. + * @see HourTime.Fields#year + */ + @Nonnull + public Integer getYear() { + if (_yearField!= null) { + return _yearField; + } else { + Object __rawValue = super._map.get("year"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("year"); + } + _yearField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _yearField; + } + } + + /** + * Setter for year + * + * @see HourTime.Fields#year + */ + public HourTime setYear(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setYear(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field year of com.linkedin.feathr.config.join.HourTime"); + } else { + CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); + _yearField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeYear(); + } else { + CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); + _yearField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); + _yearField = value; + } + break; + } + return this; + } + + /** + * Setter for year + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see HourTime.Fields#year + */ + public HourTime setYear( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field year of com.linkedin.feathr.config.join.HourTime to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); + _yearField = value; + } + return this; + } + + /** + * Setter for year + * + * @see HourTime.Fields#year + */ + public HourTime setYear(int value) { + CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); + _yearField = value; + return this; + } + + /** + * Existence checker for hour + * + * @see HourTime.Fields#hour + */ + public boolean hasHour() { + if (_hourField!= null) { + return true; + } + return super._map.containsKey("hour"); + } + + /** + * Remover for hour + * + * @see HourTime.Fields#hour + */ + public void removeHour() { + super._map.remove("hour"); + } + + /** + * Getter for hour + * + * @see HourTime.Fields#hour + */ + public Integer getHour(GetMode mode) { + switch (mode) { + case STRICT: + return getHour(); + case DEFAULT: + case NULL: + if (_hourField!= null) { + return _hourField; + } else { + Object __rawValue = super._map.get("hour"); + _hourField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _hourField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for hour + * + * @return + * Required field. Could be null for partial record. + * @see HourTime.Fields#hour + */ + @Nonnull + public Integer getHour() { + if (_hourField!= null) { + return _hourField; + } else { + Object __rawValue = super._map.get("hour"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("hour"); + } + _hourField = DataTemplateUtil.coerceIntOutput(__rawValue); + return _hourField; + } + } + + /** + * Setter for hour + * + * @see HourTime.Fields#hour + */ + public HourTime setHour(Integer value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setHour(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field hour of com.linkedin.feathr.config.join.HourTime"); + } else { + CheckedUtil.putWithoutChecking(super._map, "hour", DataTemplateUtil.coerceIntInput(value)); + _hourField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeHour(); + } else { + CheckedUtil.putWithoutChecking(super._map, "hour", DataTemplateUtil.coerceIntInput(value)); + _hourField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "hour", DataTemplateUtil.coerceIntInput(value)); + _hourField = value; + } + break; + } + return this; + } + + /** + * Setter for hour + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see HourTime.Fields#hour + */ + public HourTime setHour( + @Nonnull + Integer value) { + if (value == null) { + throw new NullPointerException("Cannot set field hour of com.linkedin.feathr.config.join.HourTime to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "hour", DataTemplateUtil.coerceIntInput(value)); + _hourField = value; + } + return this; + } + + /** + * Setter for hour + * + * @see HourTime.Fields#hour + */ + public HourTime setHour(int value) { + CheckedUtil.putWithoutChecking(super._map, "hour", DataTemplateUtil.coerceIntInput(value)); + _hourField = value; + return this; + } + + @Override + public HourTime clone() + throws CloneNotSupportedException + { + HourTime __clone = ((HourTime) super.clone()); + __clone.__changeListener = new HourTime.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public HourTime copy() + throws CloneNotSupportedException + { + HourTime __copy = ((HourTime) super.copy()); + __copy._monthField = null; + __copy._hourField = null; + __copy._yearField = null; + __copy._dayField = null; + __copy.__changeListener = new HourTime.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final HourTime __objectRef; + + private ChangeListener(HourTime reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "month": + __objectRef._monthField = null; + break; + case "hour": + __objectRef._hourField = null; + break; + case "year": + __objectRef._yearField = null; + break; + case "day": + __objectRef._dayField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * day + * + */ + public PathSpec day() { + return new PathSpec(getPathComponents(), "day"); + } + + /** + * month + * + */ + public PathSpec month() { + return new PathSpec(getPathComponents(), "month"); + } + + /** + * year + * + */ + public PathSpec year() { + return new PathSpec(getPathComponents(), "year"); + } + + /** + * hour + * + */ + public PathSpec hour() { + return new PathSpec(getPathComponents(), "hour"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(6); + } + + /** + * day + * + */ + public HourTime.ProjectionMask withDay() { + getDataMap().put("day", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * month + * + */ + public HourTime.ProjectionMask withMonth() { + getDataMap().put("month", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * year + * + */ + public HourTime.ProjectionMask withYear() { + getDataMap().put("year", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * hour + * + */ + public HourTime.ProjectionMask withHour() { + getDataMap().put("hour", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/InputDataTimeSettings.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/InputDataTimeSettings.java new file mode 100644 index 000000000..ec2c535cb --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/InputDataTimeSettings.java @@ -0,0 +1,502 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.DataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.UnionDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.UnionTemplate; + + +/** + * The data time settings pertaining to how much of the input dataset is to be loaded from the timestamp column. This is a way in which + * the input data can be restricted to allow only a fixed interval of dates to be joined with the feature data. This restriction + * will apply on the timestamp column of the input data. + * inputDataTimeSettings: { + * absoluteTimeRange: { + * startTime: Date(year=2020, month=8, day=8) + * endTime: Date(year=2020, month=8, day=10) + * } + * (or) + * relativeTimeRange: { + * offset: TimeOffset(length=1, unit="DAY") + * window: TimeWindow(length=1, unit="DAY") + * } + * } + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\InputDataTimeSettings.pdl.") +public class InputDataTimeSettings + extends RecordTemplate +{ + + private final static InputDataTimeSettings.Fields _fields = new InputDataTimeSettings.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The data time settings pertaining to how much of the input dataset is to be loaded from the timestamp column. This is a way in which\r\nthe input data can be restricted to allow only a fixed interval of dates to be joined with the feature data. This restriction\r\nwill apply on the timestamp column of the input data.\r\ninputDataTimeSettings: {\r\n absoluteTimeRange: {\r\n startTime: Date(year=2020, month=8, day=8)\r\n endTime: Date(year=2020, month=8, day=10)\r\n }\r\n (or)\r\n relativeTimeRange: {\r\n offset: TimeOffset(length=1, unit=\"DAY\")\r\n window: TimeWindow(length=1, unit=\"DAY\")\r\n }\r\n}*/record InputDataTimeSettings{/**Union of [[AbsoluteTimeRange]] and [[RelativeTimeRange]].\r\nIt indicates the range of input data which is to be loaded. This field generally refers to how much of the input\r\ndata should be restricted using the time in the timestamp column.\r\n\r\nFor example,\r\na. startDate: \"20200522\", endDate: \"20200525\" implies this feature should be joined with the input data starting from\r\n22nd May 2020 to 25th May, 2020 with both dates included.\r\nWe only support yyyyMMdd format for this. In future, if there is a request, we can\r\nadd support for other date time formats as well.\r\n\r\nb. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020\r\ntill 11/04/2020 willl be joined.*/timeRange:union[absoluteTimeRange:/**The absolute time range with start and end time being required fields.\r\nIt accepts a start time and an end time which should be specifiied using the [[Date.pdl]] or the [[HourTime.pdl]] class.\r\nThis model can be used to represent time range in daily or hourly interval.\r\nabsoluteTimeRange: {\r\n startTime: TimeHour(day=1, month=1, year=2020, hour=13)\r\n endTime: TimeHour(day=3, month=1, year=2020, hour=2)\r\n }\r\n(or)\r\nabsoluteTimeRange: {\r\n startTime: Date(day=1, month=1, year=2020)\r\n endTime: Date(day=3, month=1, year=2020)\r\n }\r\nendTime and startTime should always have the same granularity, ie - Daily or Hourly.\r\nendTme > startTime*/record AbsoluteTimeRange{/**start time of the date range, in daily or hourly format with the start date included in the range.*/startTime:union[date:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}hourTime:/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}]/**end date of the date range, in daily or hourly format with the end date included in the range.*/endTime:union[date:Date,hourTime:HourTime]}relativeTimeRange:/**The time range represented relative to the current timestamp. It uses the current system time as the reference and can be used to\r\nexpress a range of times with respect to the current time.\r\nExample, - If current time is 01/01/2020, window is 3 days, and offset is 1 day (unit can be day or hour).\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"DAY\"), offset=TimeOffset(length=1, unit=\"Day\"))\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"HOUR\"))*/record RelativeTimeRange{/**Window is the number of time units from the reference time units to look back to obtain the timeRange.\r\nFor example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020\r\ntill 11/05/2020 (both days included).\r\nwindow >= 1 TimeUnit*/window:/**Represents a length of time along with the corresponding time unit (DAY, HOUR).*/record TimeWindow{/**Amount of the duration in TimeUnits. Can be greater or equal to 1.*/@validate.positive,length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}/**Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time.\r\nFor example, if dateOffset is 4, and window is 2 days, then reference time\r\nwill be 4 days ago from today.\r\nExample - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020.\r\nThis will always take the window's timeUnits.*/@validate.integerRange.min=0,offset:long=0}]}", SchemaFormatType.PDL)); + private InputDataTimeSettings.TimeRange _timeRangeField = null; + private InputDataTimeSettings.ChangeListener __changeListener = new InputDataTimeSettings.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_TimeRange = SCHEMA.getField("timeRange"); + + public InputDataTimeSettings() { + super(new DataMap(2, 0.75F), SCHEMA, 2); + addChangeListener(__changeListener); + } + + public InputDataTimeSettings(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static InputDataTimeSettings.Fields fields() { + return _fields; + } + + public static InputDataTimeSettings.ProjectionMask createMask() { + return new InputDataTimeSettings.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for timeRange + * + * @see InputDataTimeSettings.Fields#timeRange + */ + public boolean hasTimeRange() { + if (_timeRangeField!= null) { + return true; + } + return super._map.containsKey("timeRange"); + } + + /** + * Remover for timeRange + * + * @see InputDataTimeSettings.Fields#timeRange + */ + public void removeTimeRange() { + super._map.remove("timeRange"); + } + + /** + * Getter for timeRange + * + * @see InputDataTimeSettings.Fields#timeRange + */ + public InputDataTimeSettings.TimeRange getTimeRange(GetMode mode) { + switch (mode) { + case STRICT: + return getTimeRange(); + case DEFAULT: + case NULL: + if (_timeRangeField!= null) { + return _timeRangeField; + } else { + Object __rawValue = super._map.get("timeRange"); + _timeRangeField = ((__rawValue == null)?null:new InputDataTimeSettings.TimeRange(__rawValue)); + return _timeRangeField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for timeRange + * + * @return + * Required field. Could be null for partial record. + * @see InputDataTimeSettings.Fields#timeRange + */ + @Nonnull + public InputDataTimeSettings.TimeRange getTimeRange() { + if (_timeRangeField!= null) { + return _timeRangeField; + } else { + Object __rawValue = super._map.get("timeRange"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("timeRange"); + } + _timeRangeField = ((__rawValue == null)?null:new InputDataTimeSettings.TimeRange(__rawValue)); + return _timeRangeField; + } + } + + /** + * Setter for timeRange + * + * @see InputDataTimeSettings.Fields#timeRange + */ + public InputDataTimeSettings setTimeRange(InputDataTimeSettings.TimeRange value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setTimeRange(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field timeRange of com.linkedin.feathr.config.join.InputDataTimeSettings"); + } else { + CheckedUtil.putWithoutChecking(super._map, "timeRange", value.data()); + _timeRangeField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeTimeRange(); + } else { + CheckedUtil.putWithoutChecking(super._map, "timeRange", value.data()); + _timeRangeField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "timeRange", value.data()); + _timeRangeField = value; + } + break; + } + return this; + } + + /** + * Setter for timeRange + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see InputDataTimeSettings.Fields#timeRange + */ + public InputDataTimeSettings setTimeRange( + @Nonnull + InputDataTimeSettings.TimeRange value) { + if (value == null) { + throw new NullPointerException("Cannot set field timeRange of com.linkedin.feathr.config.join.InputDataTimeSettings to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "timeRange", value.data()); + _timeRangeField = value; + } + return this; + } + + @Override + public InputDataTimeSettings clone() + throws CloneNotSupportedException + { + InputDataTimeSettings __clone = ((InputDataTimeSettings) super.clone()); + __clone.__changeListener = new InputDataTimeSettings.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public InputDataTimeSettings copy() + throws CloneNotSupportedException + { + InputDataTimeSettings __copy = ((InputDataTimeSettings) super.copy()); + __copy._timeRangeField = null; + __copy.__changeListener = new InputDataTimeSettings.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final InputDataTimeSettings __objectRef; + + private ChangeListener(InputDataTimeSettings reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "timeRange": + __objectRef._timeRangeField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Union of [[AbsoluteTimeRange]] and [[RelativeTimeRange]]. + * It indicates the range of input data which is to be loaded. This field generally refers to how much of the input + * data should be restricted using the time in the timestamp column. + * + * For example, + * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from + * 22nd May 2020 to 25th May, 2020 with both dates included. + * We only support yyyyMMdd format for this. In future, if there is a request, we can + * add support for other date time formats as well. + * + * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 + * till 11/04/2020 willl be joined. + * + */ + public com.linkedin.feathr.config.join.InputDataTimeSettings.TimeRange.Fields timeRange() { + return new com.linkedin.feathr.config.join.InputDataTimeSettings.TimeRange.Fields(getPathComponents(), "timeRange"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.InputDataTimeSettings.TimeRange.ProjectionMask _timeRangeMask; + + ProjectionMask() { + super(2); + } + + /** + * Union of [[AbsoluteTimeRange]] and [[RelativeTimeRange]]. + * It indicates the range of input data which is to be loaded. This field generally refers to how much of the input + * data should be restricted using the time in the timestamp column. + * + * For example, + * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from + * 22nd May 2020 to 25th May, 2020 with both dates included. + * We only support yyyyMMdd format for this. In future, if there is a request, we can + * add support for other date time formats as well. + * + * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 + * till 11/04/2020 willl be joined. + * + */ + public InputDataTimeSettings.ProjectionMask withTimeRange(Function nestedMask) { + _timeRangeMask = nestedMask.apply(((_timeRangeMask == null)?InputDataTimeSettings.TimeRange.createMask():_timeRangeMask)); + getDataMap().put("timeRange", _timeRangeMask.getDataMap()); + return this; + } + + /** + * Union of [[AbsoluteTimeRange]] and [[RelativeTimeRange]]. + * It indicates the range of input data which is to be loaded. This field generally refers to how much of the input + * data should be restricted using the time in the timestamp column. + * + * For example, + * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from + * 22nd May 2020 to 25th May, 2020 with both dates included. + * We only support yyyyMMdd format for this. In future, if there is a request, we can + * add support for other date time formats as well. + * + * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 + * till 11/04/2020 willl be joined. + * + */ + public InputDataTimeSettings.ProjectionMask withTimeRange() { + _timeRangeMask = null; + getDataMap().put("timeRange", MaskMap.POSITIVE_MASK); + return this; + } + + } + + @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\InputDataTimeSettings.pdl.") + public static class TimeRange + extends UnionTemplate + { + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[absoluteTimeRange:{namespace com.linkedin.feathr.config.join/**The absolute time range with start and end time being required fields.\r\nIt accepts a start time and an end time which should be specifiied using the [[Date.pdl]] or the [[HourTime.pdl]] class.\r\nThis model can be used to represent time range in daily or hourly interval.\r\nabsoluteTimeRange: {\r\n startTime: TimeHour(day=1, month=1, year=2020, hour=13)\r\n endTime: TimeHour(day=3, month=1, year=2020, hour=2)\r\n }\r\n(or)\r\nabsoluteTimeRange: {\r\n startTime: Date(day=1, month=1, year=2020)\r\n endTime: Date(day=3, month=1, year=2020)\r\n }\r\nendTime and startTime should always have the same granularity, ie - Daily or Hourly.\r\nendTme > startTime*/record AbsoluteTimeRange{/**start time of the date range, in daily or hourly format with the start date included in the range.*/startTime:union[date:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}hourTime:/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}]/**end date of the date range, in daily or hourly format with the end date included in the range.*/endTime:union[date:Date,hourTime:HourTime]}}relativeTimeRange:{namespace com.linkedin.feathr.config.join/**The time range represented relative to the current timestamp. It uses the current system time as the reference and can be used to\r\nexpress a range of times with respect to the current time.\r\nExample, - If current time is 01/01/2020, window is 3 days, and offset is 1 day (unit can be day or hour).\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"DAY\"), offset=TimeOffset(length=1, unit=\"Day\"))\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"HOUR\"))*/record RelativeTimeRange{/**Window is the number of time units from the reference time units to look back to obtain the timeRange.\r\nFor example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020\r\ntill 11/05/2020 (both days included).\r\nwindow >= 1 TimeUnit*/window:/**Represents a length of time along with the corresponding time unit (DAY, HOUR).*/record TimeWindow{/**Amount of the duration in TimeUnits. Can be greater or equal to 1.*/@validate.positive,length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}/**Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time.\r\nFor example, if dateOffset is 4, and window is 2 days, then reference time\r\nwill be 4 days ago from today.\r\nExample - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020.\r\nThis will always take the window's timeUnits.*/@validate.integerRange.min=0,offset:long=0}}]", SchemaFormatType.PDL)); + private com.linkedin.feathr.config.join.AbsoluteTimeRange _absoluteTimeRangeMember = null; + private com.linkedin.feathr.config.join.RelativeTimeRange _relativeTimeRangeMember = null; + private InputDataTimeSettings.TimeRange.ChangeListener __changeListener = new InputDataTimeSettings.TimeRange.ChangeListener(this); + private final static DataSchema MEMBER_AbsoluteTimeRange = SCHEMA.getTypeByMemberKey("absoluteTimeRange"); + private final static DataSchema MEMBER_RelativeTimeRange = SCHEMA.getTypeByMemberKey("relativeTimeRange"); + + public TimeRange() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public TimeRange(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static InputDataTimeSettings.TimeRange createWithAbsoluteTimeRange(com.linkedin.feathr.config.join.AbsoluteTimeRange value) { + InputDataTimeSettings.TimeRange newUnion = new InputDataTimeSettings.TimeRange(); + newUnion.setAbsoluteTimeRange(value); + return newUnion; + } + + public boolean isAbsoluteTimeRange() { + return memberIs("absoluteTimeRange"); + } + + public com.linkedin.feathr.config.join.AbsoluteTimeRange getAbsoluteTimeRange() { + checkNotNull(); + if (_absoluteTimeRangeMember!= null) { + return _absoluteTimeRangeMember; + } + Object __rawValue = super._map.get("absoluteTimeRange"); + _absoluteTimeRangeMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.AbsoluteTimeRange(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _absoluteTimeRangeMember; + } + + public void setAbsoluteTimeRange(com.linkedin.feathr.config.join.AbsoluteTimeRange value) { + checkNotNull(); + super._map.clear(); + _absoluteTimeRangeMember = value; + CheckedUtil.putWithoutChecking(super._map, "absoluteTimeRange", value.data()); + } + + public static InputDataTimeSettings.TimeRange createWithRelativeTimeRange(com.linkedin.feathr.config.join.RelativeTimeRange value) { + InputDataTimeSettings.TimeRange newUnion = new InputDataTimeSettings.TimeRange(); + newUnion.setRelativeTimeRange(value); + return newUnion; + } + + public boolean isRelativeTimeRange() { + return memberIs("relativeTimeRange"); + } + + public com.linkedin.feathr.config.join.RelativeTimeRange getRelativeTimeRange() { + checkNotNull(); + if (_relativeTimeRangeMember!= null) { + return _relativeTimeRangeMember; + } + Object __rawValue = super._map.get("relativeTimeRange"); + _relativeTimeRangeMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.RelativeTimeRange(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _relativeTimeRangeMember; + } + + public void setRelativeTimeRange(com.linkedin.feathr.config.join.RelativeTimeRange value) { + checkNotNull(); + super._map.clear(); + _relativeTimeRangeMember = value; + CheckedUtil.putWithoutChecking(super._map, "relativeTimeRange", value.data()); + } + + public static InputDataTimeSettings.TimeRange.ProjectionMask createMask() { + return new InputDataTimeSettings.TimeRange.ProjectionMask(); + } + + @Override + public InputDataTimeSettings.TimeRange clone() + throws CloneNotSupportedException + { + InputDataTimeSettings.TimeRange __clone = ((InputDataTimeSettings.TimeRange) super.clone()); + __clone.__changeListener = new InputDataTimeSettings.TimeRange.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public InputDataTimeSettings.TimeRange copy() + throws CloneNotSupportedException + { + InputDataTimeSettings.TimeRange __copy = ((InputDataTimeSettings.TimeRange) super.copy()); + __copy._relativeTimeRangeMember = null; + __copy._absoluteTimeRangeMember = null; + __copy.__changeListener = new InputDataTimeSettings.TimeRange.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final InputDataTimeSettings.TimeRange __objectRef; + + private ChangeListener(InputDataTimeSettings.TimeRange reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "relativeTimeRange": + __objectRef._relativeTimeRangeMember = null; + break; + case "absoluteTimeRange": + __objectRef._absoluteTimeRangeMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.config.join.AbsoluteTimeRange.Fields AbsoluteTimeRange() { + return new com.linkedin.feathr.config.join.AbsoluteTimeRange.Fields(getPathComponents(), "absoluteTimeRange"); + } + + public com.linkedin.feathr.config.join.RelativeTimeRange.Fields RelativeTimeRange() { + return new com.linkedin.feathr.config.join.RelativeTimeRange.Fields(getPathComponents(), "relativeTimeRange"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.AbsoluteTimeRange.ProjectionMask _AbsoluteTimeRangeMask; + private com.linkedin.feathr.config.join.RelativeTimeRange.ProjectionMask _RelativeTimeRangeMask; + + ProjectionMask() { + super(3); + } + + public InputDataTimeSettings.TimeRange.ProjectionMask withAbsoluteTimeRange(Function nestedMask) { + _AbsoluteTimeRangeMask = nestedMask.apply(((_AbsoluteTimeRangeMask == null)?com.linkedin.feathr.config.join.AbsoluteTimeRange.createMask():_AbsoluteTimeRangeMask)); + getDataMap().put("absoluteTimeRange", _AbsoluteTimeRangeMask.getDataMap()); + return this; + } + + public InputDataTimeSettings.TimeRange.ProjectionMask withRelativeTimeRange(Function nestedMask) { + _RelativeTimeRangeMask = nestedMask.apply(((_RelativeTimeRangeMask == null)?com.linkedin.feathr.config.join.RelativeTimeRange.createMask():_RelativeTimeRangeMask)); + getDataMap().put("relativeTimeRange", _RelativeTimeRangeMask.getDataMap()); + return this; + } + + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoinTimeSettings.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoinTimeSettings.java new file mode 100644 index 000000000..6c37ba441 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoinTimeSettings.java @@ -0,0 +1,231 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.DataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.TyperefDataSchema; +import com.linkedin.data.schema.UnionDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.HasTyperefInfo; +import com.linkedin.data.template.TyperefInfo; +import com.linkedin.data.template.UnionTemplate; + +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\JoinTimeSettings.pdl.") +public class JoinTimeSettings + extends UnionTemplate + implements HasTyperefInfo +{ + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[useLatestJoinTimeSettings:{namespace com.linkedin.feathr.config.join/**Settings needed when the input data is to be joined with the latest available feature data.\r\njoinTimeSettings: {\r\n useLatestFeatureData: true\r\n}*/record UseLatestJoinTimeSettings{/**Boolean value, if set to true, indicates that the latest available feature data is to be used for joining.\r\nWhen useLatestFeatureData is set, there should be no other time-based parameters.*/useLatestFeatureData:boolean=true}}timestampColJoinTimeSettings:{namespace com.linkedin.feathr.config.join/**Settings needed when the input data has a timestamp which should be used for the join.\r\njoinTimeSettings: {\r\n timestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }\r\n simulateTimeDelay: 1d\r\n}*/record TimestampColJoinTimeSettings{/**The timestamp column name and timeformat which should be used for joining with the feature data.\r\nRefer to [[TimestampColumn]].\r\nExample, TimestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }*/timestampColumn:/**Timestamp column of the input featureiized dataset, which is to be used for the join.\r\ntimestampColumn: {\r\n def: timestamp\r\n format: yyyyMMdd\r\n }*/record TimestampColumn{/**The definiton of the timestamp column, which can be a sql expression involving the timestamp column\r\nor just the column name\r\nExample:- definition: timestamp, timestamp + 10000000.*/definition:union[columnName:string,sparkSqlExpression:/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}]/**Format of the timestamp column. Must confer to java's timestampFormatter or can be\r\nepoch or epoch_millis.\r\nExample:- epoch, epoch_millis, yyyy/MM/dd*/format:/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string}/**An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted\r\nfrom the input data timestamp while joining with the feature data.\r\nWe do support negative time delays.*/simulateTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}}}]", SchemaFormatType.PDL)); + private com.linkedin.feathr.config.join.UseLatestJoinTimeSettings _useLatestJoinTimeSettingsMember = null; + private com.linkedin.feathr.config.join.TimestampColJoinTimeSettings _timestampColJoinTimeSettingsMember = null; + private JoinTimeSettings.ChangeListener __changeListener = new JoinTimeSettings.ChangeListener(this); + private final static DataSchema MEMBER_UseLatestJoinTimeSettings = SCHEMA.getTypeByMemberKey("useLatestJoinTimeSettings"); + private final static DataSchema MEMBER_TimestampColJoinTimeSettings = SCHEMA.getTypeByMemberKey("timestampColJoinTimeSettings"); + private final static TyperefInfo TYPEREFINFO = new JoinTimeSettings.UnionTyperefInfo(); + + public JoinTimeSettings() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public JoinTimeSettings(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static JoinTimeSettings createWithUseLatestJoinTimeSettings(com.linkedin.feathr.config.join.UseLatestJoinTimeSettings value) { + JoinTimeSettings newUnion = new JoinTimeSettings(); + newUnion.setUseLatestJoinTimeSettings(value); + return newUnion; + } + + public boolean isUseLatestJoinTimeSettings() { + return memberIs("useLatestJoinTimeSettings"); + } + + public com.linkedin.feathr.config.join.UseLatestJoinTimeSettings getUseLatestJoinTimeSettings() { + checkNotNull(); + if (_useLatestJoinTimeSettingsMember!= null) { + return _useLatestJoinTimeSettingsMember; + } + Object __rawValue = super._map.get("useLatestJoinTimeSettings"); + _useLatestJoinTimeSettingsMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.UseLatestJoinTimeSettings(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _useLatestJoinTimeSettingsMember; + } + + public void setUseLatestJoinTimeSettings(com.linkedin.feathr.config.join.UseLatestJoinTimeSettings value) { + checkNotNull(); + super._map.clear(); + _useLatestJoinTimeSettingsMember = value; + CheckedUtil.putWithoutChecking(super._map, "useLatestJoinTimeSettings", value.data()); + } + + public static JoinTimeSettings createWithTimestampColJoinTimeSettings(com.linkedin.feathr.config.join.TimestampColJoinTimeSettings value) { + JoinTimeSettings newUnion = new JoinTimeSettings(); + newUnion.setTimestampColJoinTimeSettings(value); + return newUnion; + } + + public boolean isTimestampColJoinTimeSettings() { + return memberIs("timestampColJoinTimeSettings"); + } + + public com.linkedin.feathr.config.join.TimestampColJoinTimeSettings getTimestampColJoinTimeSettings() { + checkNotNull(); + if (_timestampColJoinTimeSettingsMember!= null) { + return _timestampColJoinTimeSettingsMember; + } + Object __rawValue = super._map.get("timestampColJoinTimeSettings"); + _timestampColJoinTimeSettingsMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.TimestampColJoinTimeSettings(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _timestampColJoinTimeSettingsMember; + } + + public void setTimestampColJoinTimeSettings(com.linkedin.feathr.config.join.TimestampColJoinTimeSettings value) { + checkNotNull(); + super._map.clear(); + _timestampColJoinTimeSettingsMember = value; + CheckedUtil.putWithoutChecking(super._map, "timestampColJoinTimeSettings", value.data()); + } + + public static JoinTimeSettings.ProjectionMask createMask() { + return new JoinTimeSettings.ProjectionMask(); + } + + @Override + public JoinTimeSettings clone() + throws CloneNotSupportedException + { + JoinTimeSettings __clone = ((JoinTimeSettings) super.clone()); + __clone.__changeListener = new JoinTimeSettings.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public JoinTimeSettings copy() + throws CloneNotSupportedException + { + JoinTimeSettings __copy = ((JoinTimeSettings) super.copy()); + __copy._useLatestJoinTimeSettingsMember = null; + __copy._timestampColJoinTimeSettingsMember = null; + __copy.__changeListener = new JoinTimeSettings.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + public TyperefInfo typerefInfo() { + return TYPEREFINFO; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final JoinTimeSettings __objectRef; + + private ChangeListener(JoinTimeSettings reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "useLatestJoinTimeSettings": + __objectRef._useLatestJoinTimeSettingsMember = null; + break; + case "timestampColJoinTimeSettings": + __objectRef._timestampColJoinTimeSettingsMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.config.join.UseLatestJoinTimeSettings.Fields UseLatestJoinTimeSettings() { + return new com.linkedin.feathr.config.join.UseLatestJoinTimeSettings.Fields(getPathComponents(), "useLatestJoinTimeSettings"); + } + + public com.linkedin.feathr.config.join.TimestampColJoinTimeSettings.Fields TimestampColJoinTimeSettings() { + return new com.linkedin.feathr.config.join.TimestampColJoinTimeSettings.Fields(getPathComponents(), "timestampColJoinTimeSettings"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.UseLatestJoinTimeSettings.ProjectionMask _UseLatestJoinTimeSettingsMask; + private com.linkedin.feathr.config.join.TimestampColJoinTimeSettings.ProjectionMask _TimestampColJoinTimeSettingsMask; + + ProjectionMask() { + super(3); + } + + public JoinTimeSettings.ProjectionMask withUseLatestJoinTimeSettings(Function nestedMask) { + _UseLatestJoinTimeSettingsMask = nestedMask.apply(((_UseLatestJoinTimeSettingsMask == null)?com.linkedin.feathr.config.join.UseLatestJoinTimeSettings.createMask():_UseLatestJoinTimeSettingsMask)); + getDataMap().put("useLatestJoinTimeSettings", _UseLatestJoinTimeSettingsMask.getDataMap()); + return this; + } + + public JoinTimeSettings.ProjectionMask withTimestampColJoinTimeSettings(Function nestedMask) { + _TimestampColJoinTimeSettingsMask = nestedMask.apply(((_TimestampColJoinTimeSettingsMask == null)?com.linkedin.feathr.config.join.TimestampColJoinTimeSettings.createMask():_TimestampColJoinTimeSettingsMask)); + getDataMap().put("timestampColJoinTimeSettings", _TimestampColJoinTimeSettingsMask.getDataMap()); + return this; + } + + } + + + /** + * JoinTimeSettings contains all the parameters required to join the time sensitive input data with the feature data. + * The input data can be time sensitive in two ways:- + * a. Have a timestamp column + * b. Always join with the latest available feature data. In this case, we do not require a timestamp column. + * c. The file path is time-partition and the path time is used for the join + * (Todo - Add useTimePartitionPattern field in this section) + * In this section, the user needs to let feathr know which of the above properties is to be used for the join. + * + */ + private final static class UnionTyperefInfo + extends TyperefInfo + { + + private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**JoinTimeSettings contains all the parameters required to join the time sensitive input data with the feature data.\r\nThe input data can be time sensitive in two ways:-\r\na. Have a timestamp column\r\nb. Always join with the latest available feature data. In this case, we do not require a timestamp column.\r\nc. The file path is time-partition and the path time is used for the join\r\n(Todo - Add useTimePartitionPattern field in this section)\r\nIn this section, the user needs to let feathr know which of the above properties is to be used for the join.*/typeref JoinTimeSettings=union[useLatestJoinTimeSettings:/**Settings needed when the input data is to be joined with the latest available feature data.\r\njoinTimeSettings: {\r\n useLatestFeatureData: true\r\n}*/record UseLatestJoinTimeSettings{/**Boolean value, if set to true, indicates that the latest available feature data is to be used for joining.\r\nWhen useLatestFeatureData is set, there should be no other time-based parameters.*/useLatestFeatureData:boolean=true}timestampColJoinTimeSettings:/**Settings needed when the input data has a timestamp which should be used for the join.\r\njoinTimeSettings: {\r\n timestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }\r\n simulateTimeDelay: 1d\r\n}*/record TimestampColJoinTimeSettings{/**The timestamp column name and timeformat which should be used for joining with the feature data.\r\nRefer to [[TimestampColumn]].\r\nExample, TimestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }*/timestampColumn:/**Timestamp column of the input featureiized dataset, which is to be used for the join.\r\ntimestampColumn: {\r\n def: timestamp\r\n format: yyyyMMdd\r\n }*/record TimestampColumn{/**The definiton of the timestamp column, which can be a sql expression involving the timestamp column\r\nor just the column name\r\nExample:- definition: timestamp, timestamp + 10000000.*/definition:union[columnName:string,sparkSqlExpression:/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}]/**Format of the timestamp column. Must confer to java's timestampFormatter or can be\r\nepoch or epoch_millis.\r\nExample:- epoch, epoch_millis, yyyy/MM/dd*/format:/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string}/**An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted\r\nfrom the input data timestamp while joining with the feature data.\r\nWe do support negative time delays.*/simulateTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}}]", SchemaFormatType.PDL)); + + public UnionTyperefInfo() { + super(SCHEMA); + } + + public static TyperefDataSchema dataSchema() { + return SCHEMA; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeature.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeature.java new file mode 100644 index 000000000..6bf8b7a61 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeature.java @@ -0,0 +1,1136 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.DataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.UnionDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.StringArray; +import com.linkedin.data.template.UnionTemplate; + + +/** + * JoiningFeature is the feature section of the join config. This section consists of information pertaining to a feature + * which is to be joined:- + * a. The join keys of the input data, with which this feature is to be joined. + * b. name of the feature + * c. optional timeRange of the input data which is to be joined with this feature. + * d. optional overrideTimeDelay if this feature needs a different simulate time delay other than the one mentioned. + * + * This is a required section of the the join config. + * Example, + * a. JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature1" + * AbsoluteDateRange(startDate: Date(year=2020, month=5, day=5), + * endDate: Date(year=2020, month=5, day=7)) + * } + * b. JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature2" + * overrideTimeDelay: TimeDelay(length=1, unit="DAY") + * } + * c. JoiningFeature{ + * keys: ["key1"] + * frameFeatureName: "feature3" + * RelativeDateRange(numDays: 5, + * offset: 3) + * } + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\JoiningFeature.pdl.") +public class JoiningFeature + extends RecordTemplate +{ + + private final static JoiningFeature.Fields _fields = new JoiningFeature.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**JoiningFeature is the feature section of the join config. This section consists of information pertaining to a feature\r\nwhich is to be joined:-\r\na. The join keys of the input data, with which this feature is to be joined.\r\nb. name of the feature\r\nc. optional timeRange of the input data which is to be joined with this feature.\r\nd. optional overrideTimeDelay if this feature needs a different simulate time delay other than the one mentioned.\r\n\r\nThis is a required section of the the join config.\r\nExample,\r\n a. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature1\"\r\n AbsoluteDateRange(startDate: Date(year=2020, month=5, day=5),\r\n endDate: Date(year=2020, month=5, day=7))\r\n }\r\n b. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature2\"\r\n overrideTimeDelay: TimeDelay(length=1, unit=\"DAY\")\r\n }\r\n c. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature3\"\r\n RelativeDateRange(numDays: 5,\r\n offset: 3)\r\n }*/record JoiningFeature{/**Keys to join input with feature source, the field name of the key in the input featuized dataset.*/keys:array[string]/**Feature name as defined in feathr's feature definition configuration.\r\n\r\nCurrently the column in the output FDS that holds this feature will have the same name as feature name.\r\nIf multiple joined features have the same name and no alias is defined for them, feathr will prepend the keys to the feature name.\r\n\r\nIn the future, if \"featureAlias\" is not set, the column in the output FDS that holds this feature will have the same name as feature name.\r\nIf multiple joined features have the same name and no alias is defined for them, the join operation will fail\r\n(to avoid produciing two columns in the output FDS with the same name).*/frameFeatureName:string/**The development of this is in progress. This is not in use for now.\r\n\r\nThe name to be used for the column in the output FDS that contains the values from this joined feature.\r\nIf not set, the name of the feature (frameFeatureName) will be used for the output column.\r\nFor example, if the user request joining a feature named \"careers_job_listTime\" and provides no alias,\r\nthe output FDS will contain a column called \"careers_job_listTime\". However, if the user sets \"featureAlias\" to \"list_time\",\r\nthe column will be named \"list_time\".\r\n\r\nfeature alias can be useful for in a few cases:\r\n - If the user prefers to use a name different than the feathr name in their model,\r\nthey can use an alias to control the name of the column in the output FDS.\r\n - Sometimes, the training datas needs to have two features that are from the same feathr feature.\r\nFor example, if we are modeing the problem of the probability of a member A (viewer) seeing the profile of member B\r\n(viewee) and we want to use the skills of both viewer and viewee as features, we need to join feathr feature\r\n\"member_skills\" of member A with feathr feature \"member_skills\" of member B. That is, the two features are the same\r\nfeature but for different entiity ids). The default behavior of join is to name the output column name using the feathr\r\nfeature name, but in a case like the above case, that would result in two columns with the same name,\r\nwhich is not valid for FDS. In these cases, the user has to provide an alias for at least one of these joined features.\r\nFor example, the user can use featureAliases such as \"viewer_skills\" and \"viewee_skills\".\r\nIn these cases, featureAliases becomes mandatory.*/featureAlias:optional string/**dateRange is used in Time-based joins, which refers to the situation when one or multiple days of input data needs\r\nto be used for training.\r\nOne of the common use cases where this is used, is in training with some time-insensitive features, or\r\ntraining pipeline that always use the full day data, one day before running (since there is only partial data for today).\r\nThe time for the input featurized dataset can be set using this field.\r\nHourly data is not allowed in this case.\r\n\r\nFor example,\r\na. startDate: \"20200522\", endDate: \"20200525\" implies this feature should be joined with the input data starting from\r\n22nd May 2020 to 25th May, 2020 with both dates included.\r\nWe only support yyyyMMdd format for this. In future, if there is a request, we can\r\nadd support for other date time formats as well.\r\n\r\nb. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020\r\ntill 11/04/2020 willl be joined.\r\n\r\nP.S - This is different from the timeRange used in settings as the settings startTime is applicable for the entire input data,\r\nwhile this a feature level setting. Also, we do not support hourly time here.*/dateRange:optional union[absoluteDateRange:/**The absolute date range with start and end date being required fields.\r\nIt accepts a start date and an end date which should be specifiied using the [[Date.pdl]] class.\r\nabsoluteDateRange: {\r\n startDate: Date(day=1, month=1, year=2020)\r\n endDate: Date(day=3, month=1, year=2020)\r\n }\r\n In this case, the endDate > startDate.*/record AbsoluteDateRange{/**start date of the date range, with the start date included in the range.*/startDate:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}/**end date of the date range, with the end date included in the range.*/endDate:Date}relativeDateRange:/**The date range represented relative to the current date. It uses the current system date as the reference and can be used to\r\nexpress a range of dates with respect to the current date.\r\nExample, - If current date is 01/01/2020, window is 3, and offset 1 (unit is number of days)\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nIf dateOffset is not specified, it defaults to 0.\r\nrelativeDateRange: RelativeDateRange(numDays=2, dateOffset=1)\r\nrelativeDateRange: RelativeDateRange(numDays=5)*/record RelativeDateRange{/**Represents a length of time.\r\nnumDays is the window from the reference date to look back to obtain a dateRange.\r\nFor example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020\r\ntill 11/05/2020.*/@validate.positive={}numDays:long/**Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date\r\nwill be 4 days ago from today.*/dateOffset:long=0}]/**The override time delay parameter which will override the global simulate time delay specified in the settings section for\r\nthe particular feature.\r\nThis parameter is only applicable when the simulate time delay is set in the settings section\r\nFor example, let us say the global simulate delay was 5d, and the overrideTimeDelay is set to 3d.\r\nThen, for this specificc feature, a simulate delay of 3d will be applied.*/overrideTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}}", SchemaFormatType.PDL)); + private StringArray _keysField = null; + private String _frameFeatureNameField = null; + private String _featureAliasField = null; + private JoiningFeature.DateRange _dateRangeField = null; + private TimeOffset _overrideTimeDelayField = null; + private JoiningFeature.ChangeListener __changeListener = new JoiningFeature.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Keys = SCHEMA.getField("keys"); + private final static RecordDataSchema.Field FIELD_FrameFeatureName = SCHEMA.getField("frameFeatureName"); + private final static RecordDataSchema.Field FIELD_FeatureAlias = SCHEMA.getField("featureAlias"); + private final static RecordDataSchema.Field FIELD_DateRange = SCHEMA.getField("dateRange"); + private final static RecordDataSchema.Field FIELD_OverrideTimeDelay = SCHEMA.getField("overrideTimeDelay"); + + public JoiningFeature() { + super(new DataMap(7, 0.75F), SCHEMA, 4); + addChangeListener(__changeListener); + } + + public JoiningFeature(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static JoiningFeature.Fields fields() { + return _fields; + } + + public static JoiningFeature.ProjectionMask createMask() { + return new JoiningFeature.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for keys + * + * @see JoiningFeature.Fields#keys + */ + public boolean hasKeys() { + if (_keysField!= null) { + return true; + } + return super._map.containsKey("keys"); + } + + /** + * Remover for keys + * + * @see JoiningFeature.Fields#keys + */ + public void removeKeys() { + super._map.remove("keys"); + } + + /** + * Getter for keys + * + * @see JoiningFeature.Fields#keys + */ + public StringArray getKeys(GetMode mode) { + switch (mode) { + case STRICT: + return getKeys(); + case DEFAULT: + case NULL: + if (_keysField!= null) { + return _keysField; + } else { + Object __rawValue = super._map.get("keys"); + _keysField = ((__rawValue == null)?null:new StringArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _keysField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for keys + * + * @return + * Required field. Could be null for partial record. + * @see JoiningFeature.Fields#keys + */ + @Nonnull + public StringArray getKeys() { + if (_keysField!= null) { + return _keysField; + } else { + Object __rawValue = super._map.get("keys"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("keys"); + } + _keysField = ((__rawValue == null)?null:new StringArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); + return _keysField; + } + } + + /** + * Setter for keys + * + * @see JoiningFeature.Fields#keys + */ + public JoiningFeature setKeys(StringArray value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setKeys(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field keys of com.linkedin.feathr.config.join.JoiningFeature"); + } else { + CheckedUtil.putWithoutChecking(super._map, "keys", value.data()); + _keysField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeKeys(); + } else { + CheckedUtil.putWithoutChecking(super._map, "keys", value.data()); + _keysField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "keys", value.data()); + _keysField = value; + } + break; + } + return this; + } + + /** + * Setter for keys + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see JoiningFeature.Fields#keys + */ + public JoiningFeature setKeys( + @Nonnull + StringArray value) { + if (value == null) { + throw new NullPointerException("Cannot set field keys of com.linkedin.feathr.config.join.JoiningFeature to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "keys", value.data()); + _keysField = value; + } + return this; + } + + /** + * Existence checker for frameFeatureName + * + * @see JoiningFeature.Fields#frameFeatureName + */ + public boolean hasFrameFeatureName() { + if (_frameFeatureNameField!= null) { + return true; + } + return super._map.containsKey("frameFeatureName"); + } + + /** + * Remover for frameFeatureName + * + * @see JoiningFeature.Fields#frameFeatureName + */ + public void removeFrameFeatureName() { + super._map.remove("frameFeatureName"); + } + + /** + * Getter for frameFeatureName + * + * @see JoiningFeature.Fields#frameFeatureName + */ + public String getFrameFeatureName(GetMode mode) { + switch (mode) { + case STRICT: + return getFrameFeatureName(); + case DEFAULT: + case NULL: + if (_frameFeatureNameField!= null) { + return _frameFeatureNameField; + } else { + Object __rawValue = super._map.get("frameFeatureName"); + _frameFeatureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _frameFeatureNameField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for frameFeatureName + * + * @return + * Required field. Could be null for partial record. + * @see JoiningFeature.Fields#frameFeatureName + */ + @Nonnull + public String getFrameFeatureName() { + if (_frameFeatureNameField!= null) { + return _frameFeatureNameField; + } else { + Object __rawValue = super._map.get("frameFeatureName"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("frameFeatureName"); + } + _frameFeatureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _frameFeatureNameField; + } + } + + /** + * Setter for frameFeatureName + * + * @see JoiningFeature.Fields#frameFeatureName + */ + public JoiningFeature setFrameFeatureName(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFrameFeatureName(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field frameFeatureName of com.linkedin.feathr.config.join.JoiningFeature"); + } else { + CheckedUtil.putWithoutChecking(super._map, "frameFeatureName", value); + _frameFeatureNameField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeFrameFeatureName(); + } else { + CheckedUtil.putWithoutChecking(super._map, "frameFeatureName", value); + _frameFeatureNameField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "frameFeatureName", value); + _frameFeatureNameField = value; + } + break; + } + return this; + } + + /** + * Setter for frameFeatureName + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see JoiningFeature.Fields#frameFeatureName + */ + public JoiningFeature setFrameFeatureName( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field frameFeatureName of com.linkedin.feathr.config.join.JoiningFeature to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "frameFeatureName", value); + _frameFeatureNameField = value; + } + return this; + } + + /** + * Existence checker for featureAlias + * + * @see JoiningFeature.Fields#featureAlias + */ + public boolean hasFeatureAlias() { + if (_featureAliasField!= null) { + return true; + } + return super._map.containsKey("featureAlias"); + } + + /** + * Remover for featureAlias + * + * @see JoiningFeature.Fields#featureAlias + */ + public void removeFeatureAlias() { + super._map.remove("featureAlias"); + } + + /** + * Getter for featureAlias + * + * @see JoiningFeature.Fields#featureAlias + */ + public String getFeatureAlias(GetMode mode) { + return getFeatureAlias(); + } + + /** + * Getter for featureAlias + * + * @return + * Optional field. Always check for null. + * @see JoiningFeature.Fields#featureAlias + */ + @Nullable + public String getFeatureAlias() { + if (_featureAliasField!= null) { + return _featureAliasField; + } else { + Object __rawValue = super._map.get("featureAlias"); + _featureAliasField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _featureAliasField; + } + } + + /** + * Setter for featureAlias + * + * @see JoiningFeature.Fields#featureAlias + */ + public JoiningFeature setFeatureAlias(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFeatureAlias(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeFeatureAlias(); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureAlias", value); + _featureAliasField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "featureAlias", value); + _featureAliasField = value; + } + break; + } + return this; + } + + /** + * Setter for featureAlias + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see JoiningFeature.Fields#featureAlias + */ + public JoiningFeature setFeatureAlias( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field featureAlias of com.linkedin.feathr.config.join.JoiningFeature to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "featureAlias", value); + _featureAliasField = value; + } + return this; + } + + /** + * Existence checker for dateRange + * + * @see JoiningFeature.Fields#dateRange + */ + public boolean hasDateRange() { + if (_dateRangeField!= null) { + return true; + } + return super._map.containsKey("dateRange"); + } + + /** + * Remover for dateRange + * + * @see JoiningFeature.Fields#dateRange + */ + public void removeDateRange() { + super._map.remove("dateRange"); + } + + /** + * Getter for dateRange + * + * @see JoiningFeature.Fields#dateRange + */ + public JoiningFeature.DateRange getDateRange(GetMode mode) { + return getDateRange(); + } + + /** + * Getter for dateRange + * + * @return + * Optional field. Always check for null. + * @see JoiningFeature.Fields#dateRange + */ + @Nullable + public JoiningFeature.DateRange getDateRange() { + if (_dateRangeField!= null) { + return _dateRangeField; + } else { + Object __rawValue = super._map.get("dateRange"); + _dateRangeField = ((__rawValue == null)?null:new JoiningFeature.DateRange(__rawValue)); + return _dateRangeField; + } + } + + /** + * Setter for dateRange + * + * @see JoiningFeature.Fields#dateRange + */ + public JoiningFeature setDateRange(JoiningFeature.DateRange value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setDateRange(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeDateRange(); + } else { + CheckedUtil.putWithoutChecking(super._map, "dateRange", value.data()); + _dateRangeField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "dateRange", value.data()); + _dateRangeField = value; + } + break; + } + return this; + } + + /** + * Setter for dateRange + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see JoiningFeature.Fields#dateRange + */ + public JoiningFeature setDateRange( + @Nonnull + JoiningFeature.DateRange value) { + if (value == null) { + throw new NullPointerException("Cannot set field dateRange of com.linkedin.feathr.config.join.JoiningFeature to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "dateRange", value.data()); + _dateRangeField = value; + } + return this; + } + + /** + * Existence checker for overrideTimeDelay + * + * @see JoiningFeature.Fields#overrideTimeDelay + */ + public boolean hasOverrideTimeDelay() { + if (_overrideTimeDelayField!= null) { + return true; + } + return super._map.containsKey("overrideTimeDelay"); + } + + /** + * Remover for overrideTimeDelay + * + * @see JoiningFeature.Fields#overrideTimeDelay + */ + public void removeOverrideTimeDelay() { + super._map.remove("overrideTimeDelay"); + } + + /** + * Getter for overrideTimeDelay + * + * @see JoiningFeature.Fields#overrideTimeDelay + */ + public TimeOffset getOverrideTimeDelay(GetMode mode) { + return getOverrideTimeDelay(); + } + + /** + * Getter for overrideTimeDelay + * + * @return + * Optional field. Always check for null. + * @see JoiningFeature.Fields#overrideTimeDelay + */ + @Nullable + public TimeOffset getOverrideTimeDelay() { + if (_overrideTimeDelayField!= null) { + return _overrideTimeDelayField; + } else { + Object __rawValue = super._map.get("overrideTimeDelay"); + _overrideTimeDelayField = ((__rawValue == null)?null:new TimeOffset(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _overrideTimeDelayField; + } + } + + /** + * Setter for overrideTimeDelay + * + * @see JoiningFeature.Fields#overrideTimeDelay + */ + public JoiningFeature setOverrideTimeDelay(TimeOffset value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setOverrideTimeDelay(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeOverrideTimeDelay(); + } else { + CheckedUtil.putWithoutChecking(super._map, "overrideTimeDelay", value.data()); + _overrideTimeDelayField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "overrideTimeDelay", value.data()); + _overrideTimeDelayField = value; + } + break; + } + return this; + } + + /** + * Setter for overrideTimeDelay + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see JoiningFeature.Fields#overrideTimeDelay + */ + public JoiningFeature setOverrideTimeDelay( + @Nonnull + TimeOffset value) { + if (value == null) { + throw new NullPointerException("Cannot set field overrideTimeDelay of com.linkedin.feathr.config.join.JoiningFeature to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "overrideTimeDelay", value.data()); + _overrideTimeDelayField = value; + } + return this; + } + + @Override + public JoiningFeature clone() + throws CloneNotSupportedException + { + JoiningFeature __clone = ((JoiningFeature) super.clone()); + __clone.__changeListener = new JoiningFeature.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public JoiningFeature copy() + throws CloneNotSupportedException + { + JoiningFeature __copy = ((JoiningFeature) super.copy()); + __copy._overrideTimeDelayField = null; + __copy._featureAliasField = null; + __copy._dateRangeField = null; + __copy._keysField = null; + __copy._frameFeatureNameField = null; + __copy.__changeListener = new JoiningFeature.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final JoiningFeature __objectRef; + + private ChangeListener(JoiningFeature reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "overrideTimeDelay": + __objectRef._overrideTimeDelayField = null; + break; + case "featureAlias": + __objectRef._featureAliasField = null; + break; + case "dateRange": + __objectRef._dateRangeField = null; + break; + case "keys": + __objectRef._keysField = null; + break; + case "frameFeatureName": + __objectRef._frameFeatureNameField = null; + break; + } + } + + } + + @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\JoiningFeature.pdl.") + public static class DateRange + extends UnionTemplate + { + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[absoluteDateRange:{namespace com.linkedin.feathr.config.join/**The absolute date range with start and end date being required fields.\r\nIt accepts a start date and an end date which should be specifiied using the [[Date.pdl]] class.\r\nabsoluteDateRange: {\r\n startDate: Date(day=1, month=1, year=2020)\r\n endDate: Date(day=3, month=1, year=2020)\r\n }\r\n In this case, the endDate > startDate.*/record AbsoluteDateRange{/**start date of the date range, with the start date included in the range.*/startDate:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}/**end date of the date range, with the end date included in the range.*/endDate:Date}}relativeDateRange:{namespace com.linkedin.feathr.config.join/**The date range represented relative to the current date. It uses the current system date as the reference and can be used to\r\nexpress a range of dates with respect to the current date.\r\nExample, - If current date is 01/01/2020, window is 3, and offset 1 (unit is number of days)\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nIf dateOffset is not specified, it defaults to 0.\r\nrelativeDateRange: RelativeDateRange(numDays=2, dateOffset=1)\r\nrelativeDateRange: RelativeDateRange(numDays=5)*/record RelativeDateRange{/**Represents a length of time.\r\nnumDays is the window from the reference date to look back to obtain a dateRange.\r\nFor example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020\r\ntill 11/05/2020.*/@validate.positive={}numDays:long/**Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date\r\nwill be 4 days ago from today.*/dateOffset:long=0}}]", SchemaFormatType.PDL)); + private com.linkedin.feathr.config.join.AbsoluteDateRange _absoluteDateRangeMember = null; + private com.linkedin.feathr.config.join.RelativeDateRange _relativeDateRangeMember = null; + private JoiningFeature.DateRange.ChangeListener __changeListener = new JoiningFeature.DateRange.ChangeListener(this); + private final static DataSchema MEMBER_AbsoluteDateRange = SCHEMA.getTypeByMemberKey("absoluteDateRange"); + private final static DataSchema MEMBER_RelativeDateRange = SCHEMA.getTypeByMemberKey("relativeDateRange"); + + public DateRange() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public DateRange(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static JoiningFeature.DateRange createWithAbsoluteDateRange(com.linkedin.feathr.config.join.AbsoluteDateRange value) { + JoiningFeature.DateRange newUnion = new JoiningFeature.DateRange(); + newUnion.setAbsoluteDateRange(value); + return newUnion; + } + + public boolean isAbsoluteDateRange() { + return memberIs("absoluteDateRange"); + } + + public com.linkedin.feathr.config.join.AbsoluteDateRange getAbsoluteDateRange() { + checkNotNull(); + if (_absoluteDateRangeMember!= null) { + return _absoluteDateRangeMember; + } + Object __rawValue = super._map.get("absoluteDateRange"); + _absoluteDateRangeMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.AbsoluteDateRange(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _absoluteDateRangeMember; + } + + public void setAbsoluteDateRange(com.linkedin.feathr.config.join.AbsoluteDateRange value) { + checkNotNull(); + super._map.clear(); + _absoluteDateRangeMember = value; + CheckedUtil.putWithoutChecking(super._map, "absoluteDateRange", value.data()); + } + + public static JoiningFeature.DateRange createWithRelativeDateRange(com.linkedin.feathr.config.join.RelativeDateRange value) { + JoiningFeature.DateRange newUnion = new JoiningFeature.DateRange(); + newUnion.setRelativeDateRange(value); + return newUnion; + } + + public boolean isRelativeDateRange() { + return memberIs("relativeDateRange"); + } + + public com.linkedin.feathr.config.join.RelativeDateRange getRelativeDateRange() { + checkNotNull(); + if (_relativeDateRangeMember!= null) { + return _relativeDateRangeMember; + } + Object __rawValue = super._map.get("relativeDateRange"); + _relativeDateRangeMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.RelativeDateRange(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _relativeDateRangeMember; + } + + public void setRelativeDateRange(com.linkedin.feathr.config.join.RelativeDateRange value) { + checkNotNull(); + super._map.clear(); + _relativeDateRangeMember = value; + CheckedUtil.putWithoutChecking(super._map, "relativeDateRange", value.data()); + } + + public static JoiningFeature.DateRange.ProjectionMask createMask() { + return new JoiningFeature.DateRange.ProjectionMask(); + } + + @Override + public JoiningFeature.DateRange clone() + throws CloneNotSupportedException + { + JoiningFeature.DateRange __clone = ((JoiningFeature.DateRange) super.clone()); + __clone.__changeListener = new JoiningFeature.DateRange.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public JoiningFeature.DateRange copy() + throws CloneNotSupportedException + { + JoiningFeature.DateRange __copy = ((JoiningFeature.DateRange) super.copy()); + __copy._absoluteDateRangeMember = null; + __copy._relativeDateRangeMember = null; + __copy.__changeListener = new JoiningFeature.DateRange.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final JoiningFeature.DateRange __objectRef; + + private ChangeListener(JoiningFeature.DateRange reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "absoluteDateRange": + __objectRef._absoluteDateRangeMember = null; + break; + case "relativeDateRange": + __objectRef._relativeDateRangeMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.config.join.AbsoluteDateRange.Fields AbsoluteDateRange() { + return new com.linkedin.feathr.config.join.AbsoluteDateRange.Fields(getPathComponents(), "absoluteDateRange"); + } + + public com.linkedin.feathr.config.join.RelativeDateRange.Fields RelativeDateRange() { + return new com.linkedin.feathr.config.join.RelativeDateRange.Fields(getPathComponents(), "relativeDateRange"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.AbsoluteDateRange.ProjectionMask _AbsoluteDateRangeMask; + private com.linkedin.feathr.config.join.RelativeDateRange.ProjectionMask _RelativeDateRangeMask; + + ProjectionMask() { + super(3); + } + + public JoiningFeature.DateRange.ProjectionMask withAbsoluteDateRange(Function nestedMask) { + _AbsoluteDateRangeMask = nestedMask.apply(((_AbsoluteDateRangeMask == null)?com.linkedin.feathr.config.join.AbsoluteDateRange.createMask():_AbsoluteDateRangeMask)); + getDataMap().put("absoluteDateRange", _AbsoluteDateRangeMask.getDataMap()); + return this; + } + + public JoiningFeature.DateRange.ProjectionMask withRelativeDateRange(Function nestedMask) { + _RelativeDateRangeMask = nestedMask.apply(((_RelativeDateRangeMask == null)?com.linkedin.feathr.config.join.RelativeDateRange.createMask():_RelativeDateRangeMask)); + getDataMap().put("relativeDateRange", _RelativeDateRangeMask.getDataMap()); + return this; + } + + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Keys to join input with feature source, the field name of the key in the input featuized dataset. + * + */ + public PathSpec keys() { + return new PathSpec(getPathComponents(), "keys"); + } + + /** + * Keys to join input with feature source, the field name of the key in the input featuized dataset. + * + */ + public PathSpec keys(Integer start, Integer count) { + PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "keys"); + if (start!= null) { + arrayPathSpec.setAttribute("start", start); + } + if (count!= null) { + arrayPathSpec.setAttribute("count", count); + } + return arrayPathSpec; + } + + /** + * Feature name as defined in feathr's feature definition configuration. + * + * Currently the column in the output FDS that holds this feature will have the same name as feature name. + * If multiple joined features have the same name and no alias is defined for them, feathr will prepend the keys to the feature name. + * + * In the future, if "featureAlias" is not set, the column in the output FDS that holds this feature will have the same name as feature name. + * If multiple joined features have the same name and no alias is defined for them, the join operation will fail + * (to avoid produciing two columns in the output FDS with the same name). + * + */ + public PathSpec frameFeatureName() { + return new PathSpec(getPathComponents(), "frameFeatureName"); + } + + /** + * The development of this is in progress. This is not in use for now. + * + * The name to be used for the column in the output FDS that contains the values from this joined feature. + * If not set, the name of the feature (frameFeatureName) will be used for the output column. + * For example, if the user request joining a feature named "careers_job_listTime" and provides no alias, + * the output FDS will contain a column called "careers_job_listTime". However, if the user sets "featureAlias" to "list_time", + * the column will be named "list_time". + * + * feature alias can be useful for in a few cases: + * - If the user prefers to use a name different than the feathr name in their model, + * they can use an alias to control the name of the column in the output FDS. + * - Sometimes, the training datas needs to have two features that are from the same feathr feature. + * For example, if we are modeing the problem of the probability of a member A (viewer) seeing the profile of member B + * (viewee) and we want to use the skills of both viewer and viewee as features, we need to join feathr feature + * "member_skills" of member A with feathr feature "member_skills" of member B. That is, the two features are the same + * feature but for different entiity ids). The default behavior of join is to name the output column name using the feathr + * feature name, but in a case like the above case, that would result in two columns with the same name, + * which is not valid for FDS. In these cases, the user has to provide an alias for at least one of these joined features. + * For example, the user can use featureAliases such as "viewer_skills" and "viewee_skills". + * In these cases, featureAliases becomes mandatory. + * + */ + public PathSpec featureAlias() { + return new PathSpec(getPathComponents(), "featureAlias"); + } + + /** + * dateRange is used in Time-based joins, which refers to the situation when one or multiple days of input data needs + * to be used for training. + * One of the common use cases where this is used, is in training with some time-insensitive features, or + * training pipeline that always use the full day data, one day before running (since there is only partial data for today). + * The time for the input featurized dataset can be set using this field. + * Hourly data is not allowed in this case. + * + * For example, + * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from + * 22nd May 2020 to 25th May, 2020 with both dates included. + * We only support yyyyMMdd format for this. In future, if there is a request, we can + * add support for other date time formats as well. + * + * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 + * till 11/04/2020 willl be joined. + * + * P.S - This is different from the timeRange used in settings as the settings startTime is applicable for the entire input data, + * while this a feature level setting. Also, we do not support hourly time here. + * + */ + public com.linkedin.feathr.config.join.JoiningFeature.DateRange.Fields dateRange() { + return new com.linkedin.feathr.config.join.JoiningFeature.DateRange.Fields(getPathComponents(), "dateRange"); + } + + /** + * The override time delay parameter which will override the global simulate time delay specified in the settings section for + * the particular feature. + * This parameter is only applicable when the simulate time delay is set in the settings section + * For example, let us say the global simulate delay was 5d, and the overrideTimeDelay is set to 3d. + * Then, for this specificc feature, a simulate delay of 3d will be applied. + * + */ + public com.linkedin.feathr.config.join.TimeOffset.Fields overrideTimeDelay() { + return new com.linkedin.feathr.config.join.TimeOffset.Fields(getPathComponents(), "overrideTimeDelay"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.JoiningFeature.DateRange.ProjectionMask _dateRangeMask; + private com.linkedin.feathr.config.join.TimeOffset.ProjectionMask _overrideTimeDelayMask; + + ProjectionMask() { + super(7); + } + + /** + * Keys to join input with feature source, the field name of the key in the input featuized dataset. + * + */ + public JoiningFeature.ProjectionMask withKeys() { + getDataMap().put("keys", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Keys to join input with feature source, the field name of the key in the input featuized dataset. + * + */ + public JoiningFeature.ProjectionMask withKeys(Integer start, Integer count) { + getDataMap().put("keys", new DataMap(3)); + if (start!= null) { + getDataMap().getDataMap("keys").put("$start", start); + } + if (count!= null) { + getDataMap().getDataMap("keys").put("$count", count); + } + return this; + } + + /** + * Feature name as defined in feathr's feature definition configuration. + * + * Currently the column in the output FDS that holds this feature will have the same name as feature name. + * If multiple joined features have the same name and no alias is defined for them, feathr will prepend the keys to the feature name. + * + * In the future, if "featureAlias" is not set, the column in the output FDS that holds this feature will have the same name as feature name. + * If multiple joined features have the same name and no alias is defined for them, the join operation will fail + * (to avoid produciing two columns in the output FDS with the same name). + * + */ + public JoiningFeature.ProjectionMask withFrameFeatureName() { + getDataMap().put("frameFeatureName", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * The development of this is in progress. This is not in use for now. + * + * The name to be used for the column in the output FDS that contains the values from this joined feature. + * If not set, the name of the feature (frameFeatureName) will be used for the output column. + * For example, if the user request joining a feature named "careers_job_listTime" and provides no alias, + * the output FDS will contain a column called "careers_job_listTime". However, if the user sets "featureAlias" to "list_time", + * the column will be named "list_time". + * + * feature alias can be useful for in a few cases: + * - If the user prefers to use a name different than the feathr name in their model, + * they can use an alias to control the name of the column in the output FDS. + * - Sometimes, the training datas needs to have two features that are from the same feathr feature. + * For example, if we are modeing the problem of the probability of a member A (viewer) seeing the profile of member B + * (viewee) and we want to use the skills of both viewer and viewee as features, we need to join feathr feature + * "member_skills" of member A with feathr feature "member_skills" of member B. That is, the two features are the same + * feature but for different entiity ids). The default behavior of join is to name the output column name using the feathr + * feature name, but in a case like the above case, that would result in two columns with the same name, + * which is not valid for FDS. In these cases, the user has to provide an alias for at least one of these joined features. + * For example, the user can use featureAliases such as "viewer_skills" and "viewee_skills". + * In these cases, featureAliases becomes mandatory. + * + */ + public JoiningFeature.ProjectionMask withFeatureAlias() { + getDataMap().put("featureAlias", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * dateRange is used in Time-based joins, which refers to the situation when one or multiple days of input data needs + * to be used for training. + * One of the common use cases where this is used, is in training with some time-insensitive features, or + * training pipeline that always use the full day data, one day before running (since there is only partial data for today). + * The time for the input featurized dataset can be set using this field. + * Hourly data is not allowed in this case. + * + * For example, + * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from + * 22nd May 2020 to 25th May, 2020 with both dates included. + * We only support yyyyMMdd format for this. In future, if there is a request, we can + * add support for other date time formats as well. + * + * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 + * till 11/04/2020 willl be joined. + * + * P.S - This is different from the timeRange used in settings as the settings startTime is applicable for the entire input data, + * while this a feature level setting. Also, we do not support hourly time here. + * + */ + public JoiningFeature.ProjectionMask withDateRange(Function nestedMask) { + _dateRangeMask = nestedMask.apply(((_dateRangeMask == null)?JoiningFeature.DateRange.createMask():_dateRangeMask)); + getDataMap().put("dateRange", _dateRangeMask.getDataMap()); + return this; + } + + /** + * dateRange is used in Time-based joins, which refers to the situation when one or multiple days of input data needs + * to be used for training. + * One of the common use cases where this is used, is in training with some time-insensitive features, or + * training pipeline that always use the full day data, one day before running (since there is only partial data for today). + * The time for the input featurized dataset can be set using this field. + * Hourly data is not allowed in this case. + * + * For example, + * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from + * 22nd May 2020 to 25th May, 2020 with both dates included. + * We only support yyyyMMdd format for this. In future, if there is a request, we can + * add support for other date time formats as well. + * + * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 + * till 11/04/2020 willl be joined. + * + * P.S - This is different from the timeRange used in settings as the settings startTime is applicable for the entire input data, + * while this a feature level setting. Also, we do not support hourly time here. + * + */ + public JoiningFeature.ProjectionMask withDateRange() { + _dateRangeMask = null; + getDataMap().put("dateRange", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * The override time delay parameter which will override the global simulate time delay specified in the settings section for + * the particular feature. + * This parameter is only applicable when the simulate time delay is set in the settings section + * For example, let us say the global simulate delay was 5d, and the overrideTimeDelay is set to 3d. + * Then, for this specificc feature, a simulate delay of 3d will be applied. + * + */ + public JoiningFeature.ProjectionMask withOverrideTimeDelay(Function nestedMask) { + _overrideTimeDelayMask = nestedMask.apply(((_overrideTimeDelayMask == null)?TimeOffset.createMask():_overrideTimeDelayMask)); + getDataMap().put("overrideTimeDelay", _overrideTimeDelayMask.getDataMap()); + return this; + } + + /** + * The override time delay parameter which will override the global simulate time delay specified in the settings section for + * the particular feature. + * This parameter is only applicable when the simulate time delay is set in the settings section + * For example, let us say the global simulate delay was 5d, and the overrideTimeDelay is set to 3d. + * Then, for this specificc feature, a simulate delay of 3d will be applied. + * + */ + public JoiningFeature.ProjectionMask withOverrideTimeDelay() { + _overrideTimeDelayMask = null; + getDataMap().put("overrideTimeDelay", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeatureArray.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeatureArray.java new file mode 100644 index 000000000..0f8368c02 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeatureArray.java @@ -0,0 +1,118 @@ + +package com.linkedin.feathr.config.join; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import com.linkedin.data.DataList; +import com.linkedin.data.DataMap; +import com.linkedin.data.schema.ArrayDataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.TemplateOutputCastException; +import com.linkedin.data.template.WrappingArrayTemplate; + +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\FrameFeatureJoinConfig.pdl.") +public class JoiningFeatureArray + extends WrappingArrayTemplate +{ + + private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[{namespace com.linkedin.feathr.config.join/**JoiningFeature is the feature section of the join config. This section consists of information pertaining to a feature\r\nwhich is to be joined:-\r\na. The join keys of the input data, with which this feature is to be joined.\r\nb. name of the feature\r\nc. optional timeRange of the input data which is to be joined with this feature.\r\nd. optional overrideTimeDelay if this feature needs a different simulate time delay other than the one mentioned.\r\n\r\nThis is a required section of the the join config.\r\nExample,\r\n a. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature1\"\r\n AbsoluteDateRange(startDate: Date(year=2020, month=5, day=5),\r\n endDate: Date(year=2020, month=5, day=7))\r\n }\r\n b. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature2\"\r\n overrideTimeDelay: TimeDelay(length=1, unit=\"DAY\")\r\n }\r\n c. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature3\"\r\n RelativeDateRange(numDays: 5,\r\n offset: 3)\r\n }*/record JoiningFeature{/**Keys to join input with feature source, the field name of the key in the input featuized dataset.*/keys:array[string]/**Feature name as defined in feathr's feature definition configuration.\r\n\r\nCurrently the column in the output FDS that holds this feature will have the same name as feature name.\r\nIf multiple joined features have the same name and no alias is defined for them, feathr will prepend the keys to the feature name.\r\n\r\nIn the future, if \"featureAlias\" is not set, the column in the output FDS that holds this feature will have the same name as feature name.\r\nIf multiple joined features have the same name and no alias is defined for them, the join operation will fail\r\n(to avoid produciing two columns in the output FDS with the same name).*/frameFeatureName:string/**The development of this is in progress. This is not in use for now.\r\n\r\nThe name to be used for the column in the output FDS that contains the values from this joined feature.\r\nIf not set, the name of the feature (frameFeatureName) will be used for the output column.\r\nFor example, if the user request joining a feature named \"careers_job_listTime\" and provides no alias,\r\nthe output FDS will contain a column called \"careers_job_listTime\". However, if the user sets \"featureAlias\" to \"list_time\",\r\nthe column will be named \"list_time\".\r\n\r\nfeature alias can be useful for in a few cases:\r\n - If the user prefers to use a name different than the feathr name in their model,\r\nthey can use an alias to control the name of the column in the output FDS.\r\n - Sometimes, the training datas needs to have two features that are from the same feathr feature.\r\nFor example, if we are modeing the problem of the probability of a member A (viewer) seeing the profile of member B\r\n(viewee) and we want to use the skills of both viewer and viewee as features, we need to join feathr feature\r\n\"member_skills\" of member A with feathr feature \"member_skills\" of member B. That is, the two features are the same\r\nfeature but for different entiity ids). The default behavior of join is to name the output column name using the feathr\r\nfeature name, but in a case like the above case, that would result in two columns with the same name,\r\nwhich is not valid for FDS. In these cases, the user has to provide an alias for at least one of these joined features.\r\nFor example, the user can use featureAliases such as \"viewer_skills\" and \"viewee_skills\".\r\nIn these cases, featureAliases becomes mandatory.*/featureAlias:optional string/**dateRange is used in Time-based joins, which refers to the situation when one or multiple days of input data needs\r\nto be used for training.\r\nOne of the common use cases where this is used, is in training with some time-insensitive features, or\r\ntraining pipeline that always use the full day data, one day before running (since there is only partial data for today).\r\nThe time for the input featurized dataset can be set using this field.\r\nHourly data is not allowed in this case.\r\n\r\nFor example,\r\na. startDate: \"20200522\", endDate: \"20200525\" implies this feature should be joined with the input data starting from\r\n22nd May 2020 to 25th May, 2020 with both dates included.\r\nWe only support yyyyMMdd format for this. In future, if there is a request, we can\r\nadd support for other date time formats as well.\r\n\r\nb. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020\r\ntill 11/04/2020 willl be joined.\r\n\r\nP.S - This is different from the timeRange used in settings as the settings startTime is applicable for the entire input data,\r\nwhile this a feature level setting. Also, we do not support hourly time here.*/dateRange:optional union[absoluteDateRange:/**The absolute date range with start and end date being required fields.\r\nIt accepts a start date and an end date which should be specifiied using the [[Date.pdl]] class.\r\nabsoluteDateRange: {\r\n startDate: Date(day=1, month=1, year=2020)\r\n endDate: Date(day=3, month=1, year=2020)\r\n }\r\n In this case, the endDate > startDate.*/record AbsoluteDateRange{/**start date of the date range, with the start date included in the range.*/startDate:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}/**end date of the date range, with the end date included in the range.*/endDate:Date}relativeDateRange:/**The date range represented relative to the current date. It uses the current system date as the reference and can be used to\r\nexpress a range of dates with respect to the current date.\r\nExample, - If current date is 01/01/2020, window is 3, and offset 1 (unit is number of days)\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nIf dateOffset is not specified, it defaults to 0.\r\nrelativeDateRange: RelativeDateRange(numDays=2, dateOffset=1)\r\nrelativeDateRange: RelativeDateRange(numDays=5)*/record RelativeDateRange{/**Represents a length of time.\r\nnumDays is the window from the reference date to look back to obtain a dateRange.\r\nFor example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020\r\ntill 11/05/2020.*/@validate.positive={}numDays:long/**Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date\r\nwill be 4 days ago from today.*/dateOffset:long=0}]/**The override time delay parameter which will override the global simulate time delay specified in the settings section for\r\nthe particular feature.\r\nThis parameter is only applicable when the simulate time delay is set in the settings section\r\nFor example, let us say the global simulate delay was 5d, and the overrideTimeDelay is set to 3d.\r\nThen, for this specificc feature, a simulate delay of 3d will be applied.*/overrideTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}}}]", SchemaFormatType.PDL)); + + public JoiningFeatureArray() { + this(new DataList()); + } + + public JoiningFeatureArray(int initialCapacity) { + this(new DataList(initialCapacity)); + } + + public JoiningFeatureArray(Collection c) { + this(new DataList(c.size())); + addAll(c); + } + + public JoiningFeatureArray(DataList data) { + super(data, SCHEMA, JoiningFeature.class); + } + + public JoiningFeatureArray(JoiningFeature first, JoiningFeature... rest) { + this(new DataList((rest.length + 1))); + add(first); + addAll(Arrays.asList(rest)); + } + + public static ArrayDataSchema dataSchema() { + return SCHEMA; + } + + public static JoiningFeatureArray.ProjectionMask createMask() { + return new JoiningFeatureArray.ProjectionMask(); + } + + @Override + public JoiningFeatureArray clone() + throws CloneNotSupportedException + { + JoiningFeatureArray __clone = ((JoiningFeatureArray) super.clone()); + return __clone; + } + + @Override + public JoiningFeatureArray copy() + throws CloneNotSupportedException + { + JoiningFeatureArray __copy = ((JoiningFeatureArray) super.copy()); + return __copy; + } + + @Override + protected JoiningFeature coerceOutput(Object object) + throws TemplateOutputCastException + { + assert(object != null); + return ((object == null)?null:new JoiningFeature(DataTemplateUtil.castOrThrow(object, DataMap.class))); + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public com.linkedin.feathr.config.join.JoiningFeature.Fields items() { + return new com.linkedin.feathr.config.join.JoiningFeature.Fields(getPathComponents(), PathSpec.WILDCARD); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.JoiningFeature.ProjectionMask _itemsMask; + + ProjectionMask() { + super(4); + } + + public JoiningFeatureArray.ProjectionMask withItems(Function nestedMask) { + _itemsMask = nestedMask.apply(((_itemsMask == null)?JoiningFeature.createMask():_itemsMask)); + getDataMap().put("$*", _itemsMask.getDataMap()); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeDateRange.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeDateRange.java new file mode 100644 index 000000000..395d50bd9 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeDateRange.java @@ -0,0 +1,443 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * The date range represented relative to the current date. It uses the current system date as the reference and can be used to + * express a range of dates with respect to the current date. + * Example, - If current date is 01/01/2020, window is 3, and offset 1 (unit is number of days) + * then this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019. + * + * If dateOffset is not specified, it defaults to 0. + * relativeDateRange: RelativeDateRange(numDays=2, dateOffset=1) + * relativeDateRange: RelativeDateRange(numDays=5) + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\RelativeDateRange.pdl.") +public class RelativeDateRange + extends RecordTemplate +{ + + private final static RelativeDateRange.Fields _fields = new RelativeDateRange.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The date range represented relative to the current date. It uses the current system date as the reference and can be used to\r\nexpress a range of dates with respect to the current date.\r\nExample, - If current date is 01/01/2020, window is 3, and offset 1 (unit is number of days)\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nIf dateOffset is not specified, it defaults to 0.\r\nrelativeDateRange: RelativeDateRange(numDays=2, dateOffset=1)\r\nrelativeDateRange: RelativeDateRange(numDays=5)*/record RelativeDateRange{/**Represents a length of time.\r\nnumDays is the window from the reference date to look back to obtain a dateRange.\r\nFor example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020\r\ntill 11/05/2020.*/@validate.positive={}numDays:long/**Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date\r\nwill be 4 days ago from today.*/dateOffset:long=0}", SchemaFormatType.PDL)); + private Long _numDaysField = null; + private Long _dateOffsetField = null; + private RelativeDateRange.ChangeListener __changeListener = new RelativeDateRange.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_NumDays = SCHEMA.getField("numDays"); + private final static RecordDataSchema.Field FIELD_DateOffset = SCHEMA.getField("dateOffset"); + private final static Long DEFAULT_DateOffset; + + static { + DEFAULT_DateOffset = DataTemplateUtil.coerceLongOutput(FIELD_DateOffset.getDefault()); + } + + public RelativeDateRange() { + super(new DataMap(3, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public RelativeDateRange(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static RelativeDateRange.Fields fields() { + return _fields; + } + + public static RelativeDateRange.ProjectionMask createMask() { + return new RelativeDateRange.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for numDays + * + * @see RelativeDateRange.Fields#numDays + */ + public boolean hasNumDays() { + if (_numDaysField!= null) { + return true; + } + return super._map.containsKey("numDays"); + } + + /** + * Remover for numDays + * + * @see RelativeDateRange.Fields#numDays + */ + public void removeNumDays() { + super._map.remove("numDays"); + } + + /** + * Getter for numDays + * + * @see RelativeDateRange.Fields#numDays + */ + public Long getNumDays(GetMode mode) { + switch (mode) { + case STRICT: + return getNumDays(); + case DEFAULT: + case NULL: + if (_numDaysField!= null) { + return _numDaysField; + } else { + Object __rawValue = super._map.get("numDays"); + _numDaysField = DataTemplateUtil.coerceLongOutput(__rawValue); + return _numDaysField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for numDays + * + * @return + * Required field. Could be null for partial record. + * @see RelativeDateRange.Fields#numDays + */ + @Nonnull + public Long getNumDays() { + if (_numDaysField!= null) { + return _numDaysField; + } else { + Object __rawValue = super._map.get("numDays"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("numDays"); + } + _numDaysField = DataTemplateUtil.coerceLongOutput(__rawValue); + return _numDaysField; + } + } + + /** + * Setter for numDays + * + * @see RelativeDateRange.Fields#numDays + */ + public RelativeDateRange setNumDays(Long value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setNumDays(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field numDays of com.linkedin.feathr.config.join.RelativeDateRange"); + } else { + CheckedUtil.putWithoutChecking(super._map, "numDays", DataTemplateUtil.coerceLongInput(value)); + _numDaysField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeNumDays(); + } else { + CheckedUtil.putWithoutChecking(super._map, "numDays", DataTemplateUtil.coerceLongInput(value)); + _numDaysField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "numDays", DataTemplateUtil.coerceLongInput(value)); + _numDaysField = value; + } + break; + } + return this; + } + + /** + * Setter for numDays + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see RelativeDateRange.Fields#numDays + */ + public RelativeDateRange setNumDays( + @Nonnull + Long value) { + if (value == null) { + throw new NullPointerException("Cannot set field numDays of com.linkedin.feathr.config.join.RelativeDateRange to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "numDays", DataTemplateUtil.coerceLongInput(value)); + _numDaysField = value; + } + return this; + } + + /** + * Setter for numDays + * + * @see RelativeDateRange.Fields#numDays + */ + public RelativeDateRange setNumDays(long value) { + CheckedUtil.putWithoutChecking(super._map, "numDays", DataTemplateUtil.coerceLongInput(value)); + _numDaysField = value; + return this; + } + + /** + * Existence checker for dateOffset + * + * @see RelativeDateRange.Fields#dateOffset + */ + public boolean hasDateOffset() { + if (_dateOffsetField!= null) { + return true; + } + return super._map.containsKey("dateOffset"); + } + + /** + * Remover for dateOffset + * + * @see RelativeDateRange.Fields#dateOffset + */ + public void removeDateOffset() { + super._map.remove("dateOffset"); + } + + /** + * Getter for dateOffset + * + * @see RelativeDateRange.Fields#dateOffset + */ + public Long getDateOffset(GetMode mode) { + switch (mode) { + case STRICT: + case DEFAULT: + return getDateOffset(); + case NULL: + if (_dateOffsetField!= null) { + return _dateOffsetField; + } else { + Object __rawValue = super._map.get("dateOffset"); + _dateOffsetField = DataTemplateUtil.coerceLongOutput(__rawValue); + return _dateOffsetField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for dateOffset + * + * @return + * Required field. Could be null for partial record. + * @see RelativeDateRange.Fields#dateOffset + */ + @Nonnull + public Long getDateOffset() { + if (_dateOffsetField!= null) { + return _dateOffsetField; + } else { + Object __rawValue = super._map.get("dateOffset"); + if (__rawValue == null) { + return DEFAULT_DateOffset; + } + _dateOffsetField = DataTemplateUtil.coerceLongOutput(__rawValue); + return _dateOffsetField; + } + } + + /** + * Setter for dateOffset + * + * @see RelativeDateRange.Fields#dateOffset + */ + public RelativeDateRange setDateOffset(Long value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setDateOffset(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field dateOffset of com.linkedin.feathr.config.join.RelativeDateRange"); + } else { + CheckedUtil.putWithoutChecking(super._map, "dateOffset", DataTemplateUtil.coerceLongInput(value)); + _dateOffsetField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeDateOffset(); + } else { + CheckedUtil.putWithoutChecking(super._map, "dateOffset", DataTemplateUtil.coerceLongInput(value)); + _dateOffsetField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "dateOffset", DataTemplateUtil.coerceLongInput(value)); + _dateOffsetField = value; + } + break; + } + return this; + } + + /** + * Setter for dateOffset + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see RelativeDateRange.Fields#dateOffset + */ + public RelativeDateRange setDateOffset( + @Nonnull + Long value) { + if (value == null) { + throw new NullPointerException("Cannot set field dateOffset of com.linkedin.feathr.config.join.RelativeDateRange to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "dateOffset", DataTemplateUtil.coerceLongInput(value)); + _dateOffsetField = value; + } + return this; + } + + /** + * Setter for dateOffset + * + * @see RelativeDateRange.Fields#dateOffset + */ + public RelativeDateRange setDateOffset(long value) { + CheckedUtil.putWithoutChecking(super._map, "dateOffset", DataTemplateUtil.coerceLongInput(value)); + _dateOffsetField = value; + return this; + } + + @Override + public RelativeDateRange clone() + throws CloneNotSupportedException + { + RelativeDateRange __clone = ((RelativeDateRange) super.clone()); + __clone.__changeListener = new RelativeDateRange.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public RelativeDateRange copy() + throws CloneNotSupportedException + { + RelativeDateRange __copy = ((RelativeDateRange) super.copy()); + __copy._numDaysField = null; + __copy._dateOffsetField = null; + __copy.__changeListener = new RelativeDateRange.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final RelativeDateRange __objectRef; + + private ChangeListener(RelativeDateRange reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "numDays": + __objectRef._numDaysField = null; + break; + case "dateOffset": + __objectRef._dateOffsetField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Represents a length of time. + * numDays is the window from the reference date to look back to obtain a dateRange. + * For example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020 + * till 11/05/2020. + * + */ + public PathSpec numDays() { + return new PathSpec(getPathComponents(), "numDays"); + } + + /** + * Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date + * will be 4 days ago from today. + * + */ + public PathSpec dateOffset() { + return new PathSpec(getPathComponents(), "dateOffset"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(3); + } + + /** + * Represents a length of time. + * numDays is the window from the reference date to look back to obtain a dateRange. + * For example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020 + * till 11/05/2020. + * + */ + public RelativeDateRange.ProjectionMask withNumDays() { + getDataMap().put("numDays", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date + * will be 4 days ago from today. + * + */ + public RelativeDateRange.ProjectionMask withDateOffset() { + getDataMap().put("dateOffset", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeTimeRange.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeTimeRange.java new file mode 100644 index 000000000..30d773502 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeTimeRange.java @@ -0,0 +1,453 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * The time range represented relative to the current timestamp. It uses the current system time as the reference and can be used to + * express a range of times with respect to the current time. + * Example, - If current time is 01/01/2020, window is 3 days, and offset is 1 day (unit can be day or hour). + * then this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019. + * + * relativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit="DAY"), offset=TimeOffset(length=1, unit="Day")) + * relativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit="HOUR")) + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\RelativeTimeRange.pdl.") +public class RelativeTimeRange + extends RecordTemplate +{ + + private final static RelativeTimeRange.Fields _fields = new RelativeTimeRange.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The time range represented relative to the current timestamp. It uses the current system time as the reference and can be used to\r\nexpress a range of times with respect to the current time.\r\nExample, - If current time is 01/01/2020, window is 3 days, and offset is 1 day (unit can be day or hour).\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"DAY\"), offset=TimeOffset(length=1, unit=\"Day\"))\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"HOUR\"))*/record RelativeTimeRange{/**Window is the number of time units from the reference time units to look back to obtain the timeRange.\r\nFor example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020\r\ntill 11/05/2020 (both days included).\r\nwindow >= 1 TimeUnit*/window:/**Represents a length of time along with the corresponding time unit (DAY, HOUR).*/record TimeWindow{/**Amount of the duration in TimeUnits. Can be greater or equal to 1.*/@validate.positive,length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}/**Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time.\r\nFor example, if dateOffset is 4, and window is 2 days, then reference time\r\nwill be 4 days ago from today.\r\nExample - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020.\r\nThis will always take the window's timeUnits.*/@validate.integerRange.min=0,offset:long=0}", SchemaFormatType.PDL)); + private TimeWindow _windowField = null; + private Long _offsetField = null; + private RelativeTimeRange.ChangeListener __changeListener = new RelativeTimeRange.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Window = SCHEMA.getField("window"); + private final static RecordDataSchema.Field FIELD_Offset = SCHEMA.getField("offset"); + private final static Long DEFAULT_Offset; + + static { + DEFAULT_Offset = DataTemplateUtil.coerceLongOutput(FIELD_Offset.getDefault()); + } + + public RelativeTimeRange() { + super(new DataMap(3, 0.75F), SCHEMA, 2); + addChangeListener(__changeListener); + } + + public RelativeTimeRange(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static RelativeTimeRange.Fields fields() { + return _fields; + } + + public static RelativeTimeRange.ProjectionMask createMask() { + return new RelativeTimeRange.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for window + * + * @see RelativeTimeRange.Fields#window + */ + public boolean hasWindow() { + if (_windowField!= null) { + return true; + } + return super._map.containsKey("window"); + } + + /** + * Remover for window + * + * @see RelativeTimeRange.Fields#window + */ + public void removeWindow() { + super._map.remove("window"); + } + + /** + * Getter for window + * + * @see RelativeTimeRange.Fields#window + */ + public TimeWindow getWindow(GetMode mode) { + switch (mode) { + case STRICT: + return getWindow(); + case DEFAULT: + case NULL: + if (_windowField!= null) { + return _windowField; + } else { + Object __rawValue = super._map.get("window"); + _windowField = ((__rawValue == null)?null:new TimeWindow(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _windowField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for window + * + * @return + * Required field. Could be null for partial record. + * @see RelativeTimeRange.Fields#window + */ + @Nonnull + public TimeWindow getWindow() { + if (_windowField!= null) { + return _windowField; + } else { + Object __rawValue = super._map.get("window"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("window"); + } + _windowField = ((__rawValue == null)?null:new TimeWindow(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _windowField; + } + } + + /** + * Setter for window + * + * @see RelativeTimeRange.Fields#window + */ + public RelativeTimeRange setWindow(TimeWindow value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setWindow(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field window of com.linkedin.feathr.config.join.RelativeTimeRange"); + } else { + CheckedUtil.putWithoutChecking(super._map, "window", value.data()); + _windowField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeWindow(); + } else { + CheckedUtil.putWithoutChecking(super._map, "window", value.data()); + _windowField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "window", value.data()); + _windowField = value; + } + break; + } + return this; + } + + /** + * Setter for window + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see RelativeTimeRange.Fields#window + */ + public RelativeTimeRange setWindow( + @Nonnull + TimeWindow value) { + if (value == null) { + throw new NullPointerException("Cannot set field window of com.linkedin.feathr.config.join.RelativeTimeRange to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "window", value.data()); + _windowField = value; + } + return this; + } + + /** + * Existence checker for offset + * + * @see RelativeTimeRange.Fields#offset + */ + public boolean hasOffset() { + if (_offsetField!= null) { + return true; + } + return super._map.containsKey("offset"); + } + + /** + * Remover for offset + * + * @see RelativeTimeRange.Fields#offset + */ + public void removeOffset() { + super._map.remove("offset"); + } + + /** + * Getter for offset + * + * @see RelativeTimeRange.Fields#offset + */ + public Long getOffset(GetMode mode) { + switch (mode) { + case STRICT: + case DEFAULT: + return getOffset(); + case NULL: + if (_offsetField!= null) { + return _offsetField; + } else { + Object __rawValue = super._map.get("offset"); + _offsetField = DataTemplateUtil.coerceLongOutput(__rawValue); + return _offsetField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for offset + * + * @return + * Required field. Could be null for partial record. + * @see RelativeTimeRange.Fields#offset + */ + @Nonnull + public Long getOffset() { + if (_offsetField!= null) { + return _offsetField; + } else { + Object __rawValue = super._map.get("offset"); + if (__rawValue == null) { + return DEFAULT_Offset; + } + _offsetField = DataTemplateUtil.coerceLongOutput(__rawValue); + return _offsetField; + } + } + + /** + * Setter for offset + * + * @see RelativeTimeRange.Fields#offset + */ + public RelativeTimeRange setOffset(Long value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setOffset(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field offset of com.linkedin.feathr.config.join.RelativeTimeRange"); + } else { + CheckedUtil.putWithoutChecking(super._map, "offset", DataTemplateUtil.coerceLongInput(value)); + _offsetField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeOffset(); + } else { + CheckedUtil.putWithoutChecking(super._map, "offset", DataTemplateUtil.coerceLongInput(value)); + _offsetField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "offset", DataTemplateUtil.coerceLongInput(value)); + _offsetField = value; + } + break; + } + return this; + } + + /** + * Setter for offset + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see RelativeTimeRange.Fields#offset + */ + public RelativeTimeRange setOffset( + @Nonnull + Long value) { + if (value == null) { + throw new NullPointerException("Cannot set field offset of com.linkedin.feathr.config.join.RelativeTimeRange to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "offset", DataTemplateUtil.coerceLongInput(value)); + _offsetField = value; + } + return this; + } + + /** + * Setter for offset + * + * @see RelativeTimeRange.Fields#offset + */ + public RelativeTimeRange setOffset(long value) { + CheckedUtil.putWithoutChecking(super._map, "offset", DataTemplateUtil.coerceLongInput(value)); + _offsetField = value; + return this; + } + + @Override + public RelativeTimeRange clone() + throws CloneNotSupportedException + { + RelativeTimeRange __clone = ((RelativeTimeRange) super.clone()); + __clone.__changeListener = new RelativeTimeRange.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public RelativeTimeRange copy() + throws CloneNotSupportedException + { + RelativeTimeRange __copy = ((RelativeTimeRange) super.copy()); + __copy._offsetField = null; + __copy._windowField = null; + __copy.__changeListener = new RelativeTimeRange.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final RelativeTimeRange __objectRef; + + private ChangeListener(RelativeTimeRange reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "offset": + __objectRef._offsetField = null; + break; + case "window": + __objectRef._windowField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Window is the number of time units from the reference time units to look back to obtain the timeRange. + * For example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020 + * till 11/05/2020 (both days included). + * window >= 1 TimeUnit + * + */ + public com.linkedin.feathr.config.join.TimeWindow.Fields window() { + return new com.linkedin.feathr.config.join.TimeWindow.Fields(getPathComponents(), "window"); + } + + /** + * Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time. + * For example, if dateOffset is 4, and window is 2 days, then reference time + * will be 4 days ago from today. + * Example - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020. + * This will always take the window's timeUnits. + * + */ + public PathSpec offset() { + return new PathSpec(getPathComponents(), "offset"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.TimeWindow.ProjectionMask _windowMask; + + ProjectionMask() { + super(3); + } + + /** + * Window is the number of time units from the reference time units to look back to obtain the timeRange. + * For example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020 + * till 11/05/2020 (both days included). + * window >= 1 TimeUnit + * + */ + public RelativeTimeRange.ProjectionMask withWindow(Function nestedMask) { + _windowMask = nestedMask.apply(((_windowMask == null)?TimeWindow.createMask():_windowMask)); + getDataMap().put("window", _windowMask.getDataMap()); + return this; + } + + /** + * Window is the number of time units from the reference time units to look back to obtain the timeRange. + * For example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020 + * till 11/05/2020 (both days included). + * window >= 1 TimeUnit + * + */ + public RelativeTimeRange.ProjectionMask withWindow() { + _windowMask = null; + getDataMap().put("window", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time. + * For example, if dateOffset is 4, and window is 2 days, then reference time + * will be 4 days ago from today. + * Example - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020. + * This will always take the window's timeUnits. + * + */ + public RelativeTimeRange.ProjectionMask withOffset() { + getDataMap().put("offset", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Settings.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Settings.java new file mode 100644 index 000000000..0b259d586 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Settings.java @@ -0,0 +1,400 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.SetMode; + + +/** + * The settings section contains all the config parameters required for the joining of input dataset with the + * feature data. As of now, we have only time related parameters, but in future this can be expanded. + * This section has configs related to:- + * a. How do I load the input dataset if it is time sensitive? + * b. How do I specify the join parameters for input dataset? + * For more details - https://docs.google.com/document/d/1C6u2CKWSmOmHDQEL8Ovm5V5ZZFKhC_HdxVxU9D1F9lg/edit# + * settings: { + * inputDataTimeSettings: { + * absoluteTimeRange: { + * startTime: 20200809 + * endTime: 20200810 + * timeFormat: yyyyMMdd + * } + * } + * joinTimeSettings: { + * useLatestFeatureData: true + * } + * } + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\Settings.pdl.") +public class Settings + extends RecordTemplate +{ + + private final static Settings.Fields _fields = new Settings.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The settings section contains all the config parameters required for the joining of input dataset with the\r\nfeature data. As of now, we have only time related parameters, but in future this can be expanded.\r\nThis section has configs related to:-\r\na. How do I load the input dataset if it is time sensitive?\r\nb. How do I specify the join parameters for input dataset?\r\nFor more details - https://docs.google.com/document/d/1C6u2CKWSmOmHDQEL8Ovm5V5ZZFKhC_HdxVxU9D1F9lg/edit#\r\nsettings: {\r\n inputDataTimeSettings: {\r\n absoluteTimeRange: {\r\n startTime: 20200809\r\n endTime: 20200810\r\n timeFormat: yyyyMMdd\r\n }\r\n }\r\n joinTimeSettings: {\r\n useLatestFeatureData: true\r\n }\r\n}*/record Settings{/**Config parameters related to loading of the time sensitive input data. Contains parameters related to restricting the\r\nsize of the input data with respect to the timestamp column.*/inputDataTimeSettings:optional/**The data time settings pertaining to how much of the input dataset is to be loaded from the timestamp column. This is a way in which\r\nthe input data can be restricted to allow only a fixed interval of dates to be joined with the feature data. This restriction\r\nwill apply on the timestamp column of the input data.\r\ninputDataTimeSettings: {\r\n absoluteTimeRange: {\r\n startTime: Date(year=2020, month=8, day=8)\r\n endTime: Date(year=2020, month=8, day=10)\r\n }\r\n (or)\r\n relativeTimeRange: {\r\n offset: TimeOffset(length=1, unit=\"DAY\")\r\n window: TimeWindow(length=1, unit=\"DAY\")\r\n }\r\n}*/record InputDataTimeSettings{/**Union of [[AbsoluteTimeRange]] and [[RelativeTimeRange]].\r\nIt indicates the range of input data which is to be loaded. This field generally refers to how much of the input\r\ndata should be restricted using the time in the timestamp column.\r\n\r\nFor example,\r\na. startDate: \"20200522\", endDate: \"20200525\" implies this feature should be joined with the input data starting from\r\n22nd May 2020 to 25th May, 2020 with both dates included.\r\nWe only support yyyyMMdd format for this. In future, if there is a request, we can\r\nadd support for other date time formats as well.\r\n\r\nb. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020\r\ntill 11/04/2020 willl be joined.*/timeRange:union[absoluteTimeRange:/**The absolute time range with start and end time being required fields.\r\nIt accepts a start time and an end time which should be specifiied using the [[Date.pdl]] or the [[HourTime.pdl]] class.\r\nThis model can be used to represent time range in daily or hourly interval.\r\nabsoluteTimeRange: {\r\n startTime: TimeHour(day=1, month=1, year=2020, hour=13)\r\n endTime: TimeHour(day=3, month=1, year=2020, hour=2)\r\n }\r\n(or)\r\nabsoluteTimeRange: {\r\n startTime: Date(day=1, month=1, year=2020)\r\n endTime: Date(day=3, month=1, year=2020)\r\n }\r\nendTime and startTime should always have the same granularity, ie - Daily or Hourly.\r\nendTme > startTime*/record AbsoluteTimeRange{/**start time of the date range, in daily or hourly format with the start date included in the range.*/startTime:union[date:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}hourTime:/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}]/**end date of the date range, in daily or hourly format with the end date included in the range.*/endTime:union[date:Date,hourTime:HourTime]}relativeTimeRange:/**The time range represented relative to the current timestamp. It uses the current system time as the reference and can be used to\r\nexpress a range of times with respect to the current time.\r\nExample, - If current time is 01/01/2020, window is 3 days, and offset is 1 day (unit can be day or hour).\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"DAY\"), offset=TimeOffset(length=1, unit=\"Day\"))\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"HOUR\"))*/record RelativeTimeRange{/**Window is the number of time units from the reference time units to look back to obtain the timeRange.\r\nFor example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020\r\ntill 11/05/2020 (both days included).\r\nwindow >= 1 TimeUnit*/window:/**Represents a length of time along with the corresponding time unit (DAY, HOUR).*/record TimeWindow{/**Amount of the duration in TimeUnits. Can be greater or equal to 1.*/@validate.positive,length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}/**Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time.\r\nFor example, if dateOffset is 4, and window is 2 days, then reference time\r\nwill be 4 days ago from today.\r\nExample - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020.\r\nThis will always take the window's timeUnits.*/@validate.integerRange.min=0,offset:long=0}]}/**This contains all the parameters required to join the time sensitive input data with the feature data.*/joinTimeSettings:optional/**JoinTimeSettings contains all the parameters required to join the time sensitive input data with the feature data.\r\nThe input data can be time sensitive in two ways:-\r\na. Have a timestamp column\r\nb. Always join with the latest available feature data. In this case, we do not require a timestamp column.\r\nc. The file path is time-partition and the path time is used for the join\r\n(Todo - Add useTimePartitionPattern field in this section)\r\nIn this section, the user needs to let feathr know which of the above properties is to be used for the join.*/typeref JoinTimeSettings=union[useLatestJoinTimeSettings:/**Settings needed when the input data is to be joined with the latest available feature data.\r\njoinTimeSettings: {\r\n useLatestFeatureData: true\r\n}*/record UseLatestJoinTimeSettings{/**Boolean value, if set to true, indicates that the latest available feature data is to be used for joining.\r\nWhen useLatestFeatureData is set, there should be no other time-based parameters.*/useLatestFeatureData:boolean=true}timestampColJoinTimeSettings:/**Settings needed when the input data has a timestamp which should be used for the join.\r\njoinTimeSettings: {\r\n timestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }\r\n simulateTimeDelay: 1d\r\n}*/record TimestampColJoinTimeSettings{/**The timestamp column name and timeformat which should be used for joining with the feature data.\r\nRefer to [[TimestampColumn]].\r\nExample, TimestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }*/timestampColumn:/**Timestamp column of the input featureiized dataset, which is to be used for the join.\r\ntimestampColumn: {\r\n def: timestamp\r\n format: yyyyMMdd\r\n }*/record TimestampColumn{/**The definiton of the timestamp column, which can be a sql expression involving the timestamp column\r\nor just the column name\r\nExample:- definition: timestamp, timestamp + 10000000.*/definition:union[columnName:string,sparkSqlExpression:/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}]/**Format of the timestamp column. Must confer to java's timestampFormatter or can be\r\nepoch or epoch_millis.\r\nExample:- epoch, epoch_millis, yyyy/MM/dd*/format:/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string}/**An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted\r\nfrom the input data timestamp while joining with the feature data.\r\nWe do support negative time delays.*/simulateTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:TimeUnit}}]}", SchemaFormatType.PDL)); + private InputDataTimeSettings _inputDataTimeSettingsField = null; + private JoinTimeSettings _joinTimeSettingsField = null; + private Settings.ChangeListener __changeListener = new Settings.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_InputDataTimeSettings = SCHEMA.getField("inputDataTimeSettings"); + private final static RecordDataSchema.Field FIELD_JoinTimeSettings = SCHEMA.getField("joinTimeSettings"); + + public Settings() { + super(new DataMap(3, 0.75F), SCHEMA, 3); + addChangeListener(__changeListener); + } + + public Settings(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static Settings.Fields fields() { + return _fields; + } + + public static Settings.ProjectionMask createMask() { + return new Settings.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for inputDataTimeSettings + * + * @see Settings.Fields#inputDataTimeSettings + */ + public boolean hasInputDataTimeSettings() { + if (_inputDataTimeSettingsField!= null) { + return true; + } + return super._map.containsKey("inputDataTimeSettings"); + } + + /** + * Remover for inputDataTimeSettings + * + * @see Settings.Fields#inputDataTimeSettings + */ + public void removeInputDataTimeSettings() { + super._map.remove("inputDataTimeSettings"); + } + + /** + * Getter for inputDataTimeSettings + * + * @see Settings.Fields#inputDataTimeSettings + */ + public InputDataTimeSettings getInputDataTimeSettings(GetMode mode) { + return getInputDataTimeSettings(); + } + + /** + * Getter for inputDataTimeSettings + * + * @return + * Optional field. Always check for null. + * @see Settings.Fields#inputDataTimeSettings + */ + @Nullable + public InputDataTimeSettings getInputDataTimeSettings() { + if (_inputDataTimeSettingsField!= null) { + return _inputDataTimeSettingsField; + } else { + Object __rawValue = super._map.get("inputDataTimeSettings"); + _inputDataTimeSettingsField = ((__rawValue == null)?null:new InputDataTimeSettings(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _inputDataTimeSettingsField; + } + } + + /** + * Setter for inputDataTimeSettings + * + * @see Settings.Fields#inputDataTimeSettings + */ + public Settings setInputDataTimeSettings(InputDataTimeSettings value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setInputDataTimeSettings(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeInputDataTimeSettings(); + } else { + CheckedUtil.putWithoutChecking(super._map, "inputDataTimeSettings", value.data()); + _inputDataTimeSettingsField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "inputDataTimeSettings", value.data()); + _inputDataTimeSettingsField = value; + } + break; + } + return this; + } + + /** + * Setter for inputDataTimeSettings + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Settings.Fields#inputDataTimeSettings + */ + public Settings setInputDataTimeSettings( + @Nonnull + InputDataTimeSettings value) { + if (value == null) { + throw new NullPointerException("Cannot set field inputDataTimeSettings of com.linkedin.feathr.config.join.Settings to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "inputDataTimeSettings", value.data()); + _inputDataTimeSettingsField = value; + } + return this; + } + + /** + * Existence checker for joinTimeSettings + * + * @see Settings.Fields#joinTimeSettings + */ + public boolean hasJoinTimeSettings() { + if (_joinTimeSettingsField!= null) { + return true; + } + return super._map.containsKey("joinTimeSettings"); + } + + /** + * Remover for joinTimeSettings + * + * @see Settings.Fields#joinTimeSettings + */ + public void removeJoinTimeSettings() { + super._map.remove("joinTimeSettings"); + } + + /** + * Getter for joinTimeSettings + * + * @see Settings.Fields#joinTimeSettings + */ + public JoinTimeSettings getJoinTimeSettings(GetMode mode) { + return getJoinTimeSettings(); + } + + /** + * Getter for joinTimeSettings + * + * @return + * Optional field. Always check for null. + * @see Settings.Fields#joinTimeSettings + */ + @Nullable + public JoinTimeSettings getJoinTimeSettings() { + if (_joinTimeSettingsField!= null) { + return _joinTimeSettingsField; + } else { + Object __rawValue = super._map.get("joinTimeSettings"); + _joinTimeSettingsField = ((__rawValue == null)?null:new JoinTimeSettings(__rawValue)); + return _joinTimeSettingsField; + } + } + + /** + * Setter for joinTimeSettings + * + * @see Settings.Fields#joinTimeSettings + */ + public Settings setJoinTimeSettings(JoinTimeSettings value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setJoinTimeSettings(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeJoinTimeSettings(); + } else { + CheckedUtil.putWithoutChecking(super._map, "joinTimeSettings", value.data()); + _joinTimeSettingsField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "joinTimeSettings", value.data()); + _joinTimeSettingsField = value; + } + break; + } + return this; + } + + /** + * Setter for joinTimeSettings + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see Settings.Fields#joinTimeSettings + */ + public Settings setJoinTimeSettings( + @Nonnull + JoinTimeSettings value) { + if (value == null) { + throw new NullPointerException("Cannot set field joinTimeSettings of com.linkedin.feathr.config.join.Settings to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "joinTimeSettings", value.data()); + _joinTimeSettingsField = value; + } + return this; + } + + @Override + public Settings clone() + throws CloneNotSupportedException + { + Settings __clone = ((Settings) super.clone()); + __clone.__changeListener = new Settings.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public Settings copy() + throws CloneNotSupportedException + { + Settings __copy = ((Settings) super.copy()); + __copy._inputDataTimeSettingsField = null; + __copy._joinTimeSettingsField = null; + __copy.__changeListener = new Settings.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final Settings __objectRef; + + private ChangeListener(Settings reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "inputDataTimeSettings": + __objectRef._inputDataTimeSettingsField = null; + break; + case "joinTimeSettings": + __objectRef._joinTimeSettingsField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Config parameters related to loading of the time sensitive input data. Contains parameters related to restricting the + * size of the input data with respect to the timestamp column. + * + */ + public com.linkedin.feathr.config.join.InputDataTimeSettings.Fields inputDataTimeSettings() { + return new com.linkedin.feathr.config.join.InputDataTimeSettings.Fields(getPathComponents(), "inputDataTimeSettings"); + } + + /** + * This contains all the parameters required to join the time sensitive input data with the feature data. + * + */ + public com.linkedin.feathr.config.join.JoinTimeSettings.Fields joinTimeSettings() { + return new com.linkedin.feathr.config.join.JoinTimeSettings.Fields(getPathComponents(), "joinTimeSettings"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.InputDataTimeSettings.ProjectionMask _inputDataTimeSettingsMask; + private com.linkedin.feathr.config.join.JoinTimeSettings.ProjectionMask _joinTimeSettingsMask; + + ProjectionMask() { + super(3); + } + + /** + * Config parameters related to loading of the time sensitive input data. Contains parameters related to restricting the + * size of the input data with respect to the timestamp column. + * + */ + public Settings.ProjectionMask withInputDataTimeSettings(Function nestedMask) { + _inputDataTimeSettingsMask = nestedMask.apply(((_inputDataTimeSettingsMask == null)?InputDataTimeSettings.createMask():_inputDataTimeSettingsMask)); + getDataMap().put("inputDataTimeSettings", _inputDataTimeSettingsMask.getDataMap()); + return this; + } + + /** + * Config parameters related to loading of the time sensitive input data. Contains parameters related to restricting the + * size of the input data with respect to the timestamp column. + * + */ + public Settings.ProjectionMask withInputDataTimeSettings() { + _inputDataTimeSettingsMask = null; + getDataMap().put("inputDataTimeSettings", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * This contains all the parameters required to join the time sensitive input data with the feature data. + * + */ + public Settings.ProjectionMask withJoinTimeSettings(Function nestedMask) { + _joinTimeSettingsMask = nestedMask.apply(((_joinTimeSettingsMask == null)?JoinTimeSettings.createMask():_joinTimeSettingsMask)); + getDataMap().put("joinTimeSettings", _joinTimeSettingsMask.getDataMap()); + return this; + } + + /** + * This contains all the parameters required to join the time sensitive input data with the feature data. + * + */ + public Settings.ProjectionMask withJoinTimeSettings() { + _joinTimeSettingsMask = null; + getDataMap().put("joinTimeSettings", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/SparkSqlExpression.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/SparkSqlExpression.java new file mode 100644 index 000000000..35aeff228 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/SparkSqlExpression.java @@ -0,0 +1,260 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * An expression in Spark SQL. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\SparkSqlExpression.pdl.") +public class SparkSqlExpression + extends RecordTemplate +{ + + private final static SparkSqlExpression.Fields _fields = new SparkSqlExpression.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}", SchemaFormatType.PDL)); + private String _expressionField = null; + private SparkSqlExpression.ChangeListener __changeListener = new SparkSqlExpression.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Expression = SCHEMA.getField("expression"); + + public SparkSqlExpression() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public SparkSqlExpression(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static SparkSqlExpression.Fields fields() { + return _fields; + } + + public static SparkSqlExpression.ProjectionMask createMask() { + return new SparkSqlExpression.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for expression + * + * @see SparkSqlExpression.Fields#expression + */ + public boolean hasExpression() { + if (_expressionField!= null) { + return true; + } + return super._map.containsKey("expression"); + } + + /** + * Remover for expression + * + * @see SparkSqlExpression.Fields#expression + */ + public void removeExpression() { + super._map.remove("expression"); + } + + /** + * Getter for expression + * + * @see SparkSqlExpression.Fields#expression + */ + public String getExpression(GetMode mode) { + switch (mode) { + case STRICT: + return getExpression(); + case DEFAULT: + case NULL: + if (_expressionField!= null) { + return _expressionField; + } else { + Object __rawValue = super._map.get("expression"); + _expressionField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _expressionField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for expression + * + * @return + * Required field. Could be null for partial record. + * @see SparkSqlExpression.Fields#expression + */ + @Nonnull + public String getExpression() { + if (_expressionField!= null) { + return _expressionField; + } else { + Object __rawValue = super._map.get("expression"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("expression"); + } + _expressionField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _expressionField; + } + } + + /** + * Setter for expression + * + * @see SparkSqlExpression.Fields#expression + */ + public SparkSqlExpression setExpression(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setExpression(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field expression of com.linkedin.feathr.config.join.SparkSqlExpression"); + } else { + CheckedUtil.putWithoutChecking(super._map, "expression", value); + _expressionField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeExpression(); + } else { + CheckedUtil.putWithoutChecking(super._map, "expression", value); + _expressionField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "expression", value); + _expressionField = value; + } + break; + } + return this; + } + + /** + * Setter for expression + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see SparkSqlExpression.Fields#expression + */ + public SparkSqlExpression setExpression( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field expression of com.linkedin.feathr.config.join.SparkSqlExpression to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "expression", value); + _expressionField = value; + } + return this; + } + + @Override + public SparkSqlExpression clone() + throws CloneNotSupportedException + { + SparkSqlExpression __clone = ((SparkSqlExpression) super.clone()); + __clone.__changeListener = new SparkSqlExpression.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public SparkSqlExpression copy() + throws CloneNotSupportedException + { + SparkSqlExpression __copy = ((SparkSqlExpression) super.copy()); + __copy._expressionField = null; + __copy.__changeListener = new SparkSqlExpression.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final SparkSqlExpression __objectRef; + + private ChangeListener(SparkSqlExpression reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "expression": + __objectRef._expressionField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The Spark SQL expression. + * + */ + public PathSpec expression() { + return new PathSpec(getPathComponents(), "expression"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(2); + } + + /** + * The Spark SQL expression. + * + */ + public SparkSqlExpression.ProjectionMask withExpression() { + getDataMap().put("expression", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeFormat.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeFormat.java new file mode 100644 index 000000000..3ff7d5455 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeFormat.java @@ -0,0 +1,31 @@ + +package com.linkedin.feathr.config.join; + +import javax.annotation.Generated; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.TyperefDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.TyperefInfo; + + +/** + * The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have + * the option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimeFormat.pdl.") +public class TimeFormat + extends TyperefInfo +{ + + private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string", SchemaFormatType.PDL)); + + public TimeFormat() { + super(SCHEMA); + } + + public static TyperefDataSchema dataSchema() { + return SCHEMA; + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeOffset.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeOffset.java new file mode 100644 index 000000000..4132a962f --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeOffset.java @@ -0,0 +1,414 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can + * be any time in the past also, we do allow a positive or negative offset length. + * offset - 1 day implies the previous from the reference day. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimeOffset.pdl.") +public class TimeOffset + extends RecordTemplate +{ + + private final static TimeOffset.Fields _fields = new TimeOffset.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}", SchemaFormatType.PDL)); + private Long _lengthField = null; + private TimeUnit _unitField = null; + private TimeOffset.ChangeListener __changeListener = new TimeOffset.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Length = SCHEMA.getField("length"); + private final static RecordDataSchema.Field FIELD_Unit = SCHEMA.getField("unit"); + + public TimeOffset() { + super(new DataMap(3, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public TimeOffset(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static TimeOffset.Fields fields() { + return _fields; + } + + public static TimeOffset.ProjectionMask createMask() { + return new TimeOffset.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for length + * + * @see TimeOffset.Fields#length + */ + public boolean hasLength() { + if (_lengthField!= null) { + return true; + } + return super._map.containsKey("length"); + } + + /** + * Remover for length + * + * @see TimeOffset.Fields#length + */ + public void removeLength() { + super._map.remove("length"); + } + + /** + * Getter for length + * + * @see TimeOffset.Fields#length + */ + public Long getLength(GetMode mode) { + switch (mode) { + case STRICT: + return getLength(); + case DEFAULT: + case NULL: + if (_lengthField!= null) { + return _lengthField; + } else { + Object __rawValue = super._map.get("length"); + _lengthField = DataTemplateUtil.coerceLongOutput(__rawValue); + return _lengthField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for length + * + * @return + * Required field. Could be null for partial record. + * @see TimeOffset.Fields#length + */ + @Nonnull + public Long getLength() { + if (_lengthField!= null) { + return _lengthField; + } else { + Object __rawValue = super._map.get("length"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("length"); + } + _lengthField = DataTemplateUtil.coerceLongOutput(__rawValue); + return _lengthField; + } + } + + /** + * Setter for length + * + * @see TimeOffset.Fields#length + */ + public TimeOffset setLength(Long value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setLength(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field length of com.linkedin.feathr.config.join.TimeOffset"); + } else { + CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); + _lengthField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeLength(); + } else { + CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); + _lengthField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); + _lengthField = value; + } + break; + } + return this; + } + + /** + * Setter for length + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TimeOffset.Fields#length + */ + public TimeOffset setLength( + @Nonnull + Long value) { + if (value == null) { + throw new NullPointerException("Cannot set field length of com.linkedin.feathr.config.join.TimeOffset to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); + _lengthField = value; + } + return this; + } + + /** + * Setter for length + * + * @see TimeOffset.Fields#length + */ + public TimeOffset setLength(long value) { + CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); + _lengthField = value; + return this; + } + + /** + * Existence checker for unit + * + * @see TimeOffset.Fields#unit + */ + public boolean hasUnit() { + if (_unitField!= null) { + return true; + } + return super._map.containsKey("unit"); + } + + /** + * Remover for unit + * + * @see TimeOffset.Fields#unit + */ + public void removeUnit() { + super._map.remove("unit"); + } + + /** + * Getter for unit + * + * @see TimeOffset.Fields#unit + */ + public TimeUnit getUnit(GetMode mode) { + switch (mode) { + case STRICT: + return getUnit(); + case DEFAULT: + case NULL: + if (_unitField!= null) { + return _unitField; + } else { + Object __rawValue = super._map.get("unit"); + _unitField = DataTemplateUtil.coerceEnumOutput(__rawValue, TimeUnit.class, TimeUnit.$UNKNOWN); + return _unitField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for unit + * + * @return + * Required field. Could be null for partial record. + * @see TimeOffset.Fields#unit + */ + @Nonnull + public TimeUnit getUnit() { + if (_unitField!= null) { + return _unitField; + } else { + Object __rawValue = super._map.get("unit"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("unit"); + } + _unitField = DataTemplateUtil.coerceEnumOutput(__rawValue, TimeUnit.class, TimeUnit.$UNKNOWN); + return _unitField; + } + } + + /** + * Setter for unit + * + * @see TimeOffset.Fields#unit + */ + public TimeOffset setUnit(TimeUnit value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setUnit(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field unit of com.linkedin.feathr.config.join.TimeOffset"); + } else { + CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); + _unitField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeUnit(); + } else { + CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); + _unitField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); + _unitField = value; + } + break; + } + return this; + } + + /** + * Setter for unit + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TimeOffset.Fields#unit + */ + public TimeOffset setUnit( + @Nonnull + TimeUnit value) { + if (value == null) { + throw new NullPointerException("Cannot set field unit of com.linkedin.feathr.config.join.TimeOffset to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); + _unitField = value; + } + return this; + } + + @Override + public TimeOffset clone() + throws CloneNotSupportedException + { + TimeOffset __clone = ((TimeOffset) super.clone()); + __clone.__changeListener = new TimeOffset.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public TimeOffset copy() + throws CloneNotSupportedException + { + TimeOffset __copy = ((TimeOffset) super.copy()); + __copy._unitField = null; + __copy._lengthField = null; + __copy.__changeListener = new TimeOffset.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final TimeOffset __objectRef; + + private ChangeListener(TimeOffset reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "unit": + __objectRef._unitField = null; + break; + case "length": + __objectRef._lengthField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Amount of the duration in TimeUnits. Can be positive or negative. + * + */ + public PathSpec length() { + return new PathSpec(getPathComponents(), "length"); + } + + /** + * Time unit for "length". For example, TimeUnit.DAY or TimeUnit.HOUR. + * + */ + public PathSpec unit() { + return new PathSpec(getPathComponents(), "unit"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(3); + } + + /** + * Amount of the duration in TimeUnits. Can be positive or negative. + * + */ + public TimeOffset.ProjectionMask withLength() { + getDataMap().put("length", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Time unit for "length". For example, TimeUnit.DAY or TimeUnit.HOUR. + * + */ + public TimeOffset.ProjectionMask withUnit() { + getDataMap().put("unit", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeUnit.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeUnit.java new file mode 100644 index 000000000..b5ea832a6 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeUnit.java @@ -0,0 +1,48 @@ + +package com.linkedin.feathr.config.join; + +import javax.annotation.Generated; +import com.linkedin.data.schema.EnumDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; + + +/** + * Unit of time used for defining a time range. + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimeUnit.pdl.") +public enum TimeUnit { + + + /** + * Daily format + * + */ + DAY, + + /** + * Hourly format + * + */ + HOUR, + + /** + * minute format, this can be used in simulate time delay + * + */ + MINUTE, + + /** + * second format, this can be used in simulate time delay + * + */ + SECOND, + $UNKNOWN; + private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}", SchemaFormatType.PDL)); + + public static EnumDataSchema dataSchema() { + return SCHEMA; + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeWindow.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeWindow.java new file mode 100644 index 000000000..16e3aa965 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeWindow.java @@ -0,0 +1,412 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * Represents a length of time along with the corresponding time unit (DAY, HOUR). + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimeWindow.pdl.") +public class TimeWindow + extends RecordTemplate +{ + + private final static TimeWindow.Fields _fields = new TimeWindow.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Represents a length of time along with the corresponding time unit (DAY, HOUR).*/record TimeWindow{/**Amount of the duration in TimeUnits. Can be greater or equal to 1.*/@validate.positive,length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}", SchemaFormatType.PDL)); + private Long _lengthField = null; + private TimeUnit _unitField = null; + private TimeWindow.ChangeListener __changeListener = new TimeWindow.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Length = SCHEMA.getField("length"); + private final static RecordDataSchema.Field FIELD_Unit = SCHEMA.getField("unit"); + + public TimeWindow() { + super(new DataMap(3, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public TimeWindow(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static TimeWindow.Fields fields() { + return _fields; + } + + public static TimeWindow.ProjectionMask createMask() { + return new TimeWindow.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for length + * + * @see TimeWindow.Fields#length + */ + public boolean hasLength() { + if (_lengthField!= null) { + return true; + } + return super._map.containsKey("length"); + } + + /** + * Remover for length + * + * @see TimeWindow.Fields#length + */ + public void removeLength() { + super._map.remove("length"); + } + + /** + * Getter for length + * + * @see TimeWindow.Fields#length + */ + public Long getLength(GetMode mode) { + switch (mode) { + case STRICT: + return getLength(); + case DEFAULT: + case NULL: + if (_lengthField!= null) { + return _lengthField; + } else { + Object __rawValue = super._map.get("length"); + _lengthField = DataTemplateUtil.coerceLongOutput(__rawValue); + return _lengthField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for length + * + * @return + * Required field. Could be null for partial record. + * @see TimeWindow.Fields#length + */ + @Nonnull + public Long getLength() { + if (_lengthField!= null) { + return _lengthField; + } else { + Object __rawValue = super._map.get("length"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("length"); + } + _lengthField = DataTemplateUtil.coerceLongOutput(__rawValue); + return _lengthField; + } + } + + /** + * Setter for length + * + * @see TimeWindow.Fields#length + */ + public TimeWindow setLength(Long value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setLength(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field length of com.linkedin.feathr.config.join.TimeWindow"); + } else { + CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); + _lengthField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeLength(); + } else { + CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); + _lengthField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); + _lengthField = value; + } + break; + } + return this; + } + + /** + * Setter for length + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TimeWindow.Fields#length + */ + public TimeWindow setLength( + @Nonnull + Long value) { + if (value == null) { + throw new NullPointerException("Cannot set field length of com.linkedin.feathr.config.join.TimeWindow to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); + _lengthField = value; + } + return this; + } + + /** + * Setter for length + * + * @see TimeWindow.Fields#length + */ + public TimeWindow setLength(long value) { + CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); + _lengthField = value; + return this; + } + + /** + * Existence checker for unit + * + * @see TimeWindow.Fields#unit + */ + public boolean hasUnit() { + if (_unitField!= null) { + return true; + } + return super._map.containsKey("unit"); + } + + /** + * Remover for unit + * + * @see TimeWindow.Fields#unit + */ + public void removeUnit() { + super._map.remove("unit"); + } + + /** + * Getter for unit + * + * @see TimeWindow.Fields#unit + */ + public TimeUnit getUnit(GetMode mode) { + switch (mode) { + case STRICT: + return getUnit(); + case DEFAULT: + case NULL: + if (_unitField!= null) { + return _unitField; + } else { + Object __rawValue = super._map.get("unit"); + _unitField = DataTemplateUtil.coerceEnumOutput(__rawValue, TimeUnit.class, TimeUnit.$UNKNOWN); + return _unitField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for unit + * + * @return + * Required field. Could be null for partial record. + * @see TimeWindow.Fields#unit + */ + @Nonnull + public TimeUnit getUnit() { + if (_unitField!= null) { + return _unitField; + } else { + Object __rawValue = super._map.get("unit"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("unit"); + } + _unitField = DataTemplateUtil.coerceEnumOutput(__rawValue, TimeUnit.class, TimeUnit.$UNKNOWN); + return _unitField; + } + } + + /** + * Setter for unit + * + * @see TimeWindow.Fields#unit + */ + public TimeWindow setUnit(TimeUnit value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setUnit(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field unit of com.linkedin.feathr.config.join.TimeWindow"); + } else { + CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); + _unitField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeUnit(); + } else { + CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); + _unitField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); + _unitField = value; + } + break; + } + return this; + } + + /** + * Setter for unit + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TimeWindow.Fields#unit + */ + public TimeWindow setUnit( + @Nonnull + TimeUnit value) { + if (value == null) { + throw new NullPointerException("Cannot set field unit of com.linkedin.feathr.config.join.TimeWindow to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); + _unitField = value; + } + return this; + } + + @Override + public TimeWindow clone() + throws CloneNotSupportedException + { + TimeWindow __clone = ((TimeWindow) super.clone()); + __clone.__changeListener = new TimeWindow.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public TimeWindow copy() + throws CloneNotSupportedException + { + TimeWindow __copy = ((TimeWindow) super.copy()); + __copy._unitField = null; + __copy._lengthField = null; + __copy.__changeListener = new TimeWindow.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final TimeWindow __objectRef; + + private ChangeListener(TimeWindow reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "unit": + __objectRef._unitField = null; + break; + case "length": + __objectRef._lengthField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Amount of the duration in TimeUnits. Can be greater or equal to 1. + * + */ + public PathSpec length() { + return new PathSpec(getPathComponents(), "length"); + } + + /** + * Time unit for "length". For example, TimeUnit.DAY or TimeUnit.HOUR. + * + */ + public PathSpec unit() { + return new PathSpec(getPathComponents(), "unit"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(3); + } + + /** + * Amount of the duration in TimeUnits. Can be greater or equal to 1. + * + */ + public TimeWindow.ProjectionMask withLength() { + getDataMap().put("length", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Time unit for "length". For example, TimeUnit.DAY or TimeUnit.HOUR. + * + */ + public TimeWindow.ProjectionMask withUnit() { + getDataMap().put("unit", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColJoinTimeSettings.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColJoinTimeSettings.java new file mode 100644 index 000000000..3e87c616d --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColJoinTimeSettings.java @@ -0,0 +1,432 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; + + +/** + * Settings needed when the input data has a timestamp which should be used for the join. + * joinTimeSettings: { + * timestampColumn: { + * def: timestamp + * format: yyyy/MM/dd + * } + * simulateTimeDelay: 1d + * } + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimestampColJoinTimeSettings.pdl.") +public class TimestampColJoinTimeSettings + extends RecordTemplate +{ + + private final static TimestampColJoinTimeSettings.Fields _fields = new TimestampColJoinTimeSettings.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Settings needed when the input data has a timestamp which should be used for the join.\r\njoinTimeSettings: {\r\n timestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }\r\n simulateTimeDelay: 1d\r\n}*/record TimestampColJoinTimeSettings{/**The timestamp column name and timeformat which should be used for joining with the feature data.\r\nRefer to [[TimestampColumn]].\r\nExample, TimestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }*/timestampColumn:/**Timestamp column of the input featureiized dataset, which is to be used for the join.\r\ntimestampColumn: {\r\n def: timestamp\r\n format: yyyyMMdd\r\n }*/record TimestampColumn{/**The definiton of the timestamp column, which can be a sql expression involving the timestamp column\r\nor just the column name\r\nExample:- definition: timestamp, timestamp + 10000000.*/definition:union[columnName:string,sparkSqlExpression:/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}]/**Format of the timestamp column. Must confer to java's timestampFormatter or can be\r\nepoch or epoch_millis.\r\nExample:- epoch, epoch_millis, yyyy/MM/dd*/format:/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string}/**An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted\r\nfrom the input data timestamp while joining with the feature data.\r\nWe do support negative time delays.*/simulateTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}}", SchemaFormatType.PDL)); + private TimestampColumn _timestampColumnField = null; + private TimeOffset _simulateTimeDelayField = null; + private TimestampColJoinTimeSettings.ChangeListener __changeListener = new TimestampColJoinTimeSettings.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_TimestampColumn = SCHEMA.getField("timestampColumn"); + private final static RecordDataSchema.Field FIELD_SimulateTimeDelay = SCHEMA.getField("simulateTimeDelay"); + + public TimestampColJoinTimeSettings() { + super(new DataMap(3, 0.75F), SCHEMA, 3); + addChangeListener(__changeListener); + } + + public TimestampColJoinTimeSettings(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static TimestampColJoinTimeSettings.Fields fields() { + return _fields; + } + + public static TimestampColJoinTimeSettings.ProjectionMask createMask() { + return new TimestampColJoinTimeSettings.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for timestampColumn + * + * @see TimestampColJoinTimeSettings.Fields#timestampColumn + */ + public boolean hasTimestampColumn() { + if (_timestampColumnField!= null) { + return true; + } + return super._map.containsKey("timestampColumn"); + } + + /** + * Remover for timestampColumn + * + * @see TimestampColJoinTimeSettings.Fields#timestampColumn + */ + public void removeTimestampColumn() { + super._map.remove("timestampColumn"); + } + + /** + * Getter for timestampColumn + * + * @see TimestampColJoinTimeSettings.Fields#timestampColumn + */ + public TimestampColumn getTimestampColumn(GetMode mode) { + switch (mode) { + case STRICT: + return getTimestampColumn(); + case DEFAULT: + case NULL: + if (_timestampColumnField!= null) { + return _timestampColumnField; + } else { + Object __rawValue = super._map.get("timestampColumn"); + _timestampColumnField = ((__rawValue == null)?null:new TimestampColumn(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _timestampColumnField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for timestampColumn + * + * @return + * Required field. Could be null for partial record. + * @see TimestampColJoinTimeSettings.Fields#timestampColumn + */ + @Nonnull + public TimestampColumn getTimestampColumn() { + if (_timestampColumnField!= null) { + return _timestampColumnField; + } else { + Object __rawValue = super._map.get("timestampColumn"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("timestampColumn"); + } + _timestampColumnField = ((__rawValue == null)?null:new TimestampColumn(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _timestampColumnField; + } + } + + /** + * Setter for timestampColumn + * + * @see TimestampColJoinTimeSettings.Fields#timestampColumn + */ + public TimestampColJoinTimeSettings setTimestampColumn(TimestampColumn value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setTimestampColumn(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field timestampColumn of com.linkedin.feathr.config.join.TimestampColJoinTimeSettings"); + } else { + CheckedUtil.putWithoutChecking(super._map, "timestampColumn", value.data()); + _timestampColumnField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeTimestampColumn(); + } else { + CheckedUtil.putWithoutChecking(super._map, "timestampColumn", value.data()); + _timestampColumnField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "timestampColumn", value.data()); + _timestampColumnField = value; + } + break; + } + return this; + } + + /** + * Setter for timestampColumn + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TimestampColJoinTimeSettings.Fields#timestampColumn + */ + public TimestampColJoinTimeSettings setTimestampColumn( + @Nonnull + TimestampColumn value) { + if (value == null) { + throw new NullPointerException("Cannot set field timestampColumn of com.linkedin.feathr.config.join.TimestampColJoinTimeSettings to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "timestampColumn", value.data()); + _timestampColumnField = value; + } + return this; + } + + /** + * Existence checker for simulateTimeDelay + * + * @see TimestampColJoinTimeSettings.Fields#simulateTimeDelay + */ + public boolean hasSimulateTimeDelay() { + if (_simulateTimeDelayField!= null) { + return true; + } + return super._map.containsKey("simulateTimeDelay"); + } + + /** + * Remover for simulateTimeDelay + * + * @see TimestampColJoinTimeSettings.Fields#simulateTimeDelay + */ + public void removeSimulateTimeDelay() { + super._map.remove("simulateTimeDelay"); + } + + /** + * Getter for simulateTimeDelay + * + * @see TimestampColJoinTimeSettings.Fields#simulateTimeDelay + */ + public TimeOffset getSimulateTimeDelay(GetMode mode) { + return getSimulateTimeDelay(); + } + + /** + * Getter for simulateTimeDelay + * + * @return + * Optional field. Always check for null. + * @see TimestampColJoinTimeSettings.Fields#simulateTimeDelay + */ + @Nullable + public TimeOffset getSimulateTimeDelay() { + if (_simulateTimeDelayField!= null) { + return _simulateTimeDelayField; + } else { + Object __rawValue = super._map.get("simulateTimeDelay"); + _simulateTimeDelayField = ((__rawValue == null)?null:new TimeOffset(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _simulateTimeDelayField; + } + } + + /** + * Setter for simulateTimeDelay + * + * @see TimestampColJoinTimeSettings.Fields#simulateTimeDelay + */ + public TimestampColJoinTimeSettings setSimulateTimeDelay(TimeOffset value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setSimulateTimeDelay(value); + case REMOVE_OPTIONAL_IF_NULL: + case REMOVE_IF_NULL: + if (value == null) { + removeSimulateTimeDelay(); + } else { + CheckedUtil.putWithoutChecking(super._map, "simulateTimeDelay", value.data()); + _simulateTimeDelayField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "simulateTimeDelay", value.data()); + _simulateTimeDelayField = value; + } + break; + } + return this; + } + + /** + * Setter for simulateTimeDelay + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TimestampColJoinTimeSettings.Fields#simulateTimeDelay + */ + public TimestampColJoinTimeSettings setSimulateTimeDelay( + @Nonnull + TimeOffset value) { + if (value == null) { + throw new NullPointerException("Cannot set field simulateTimeDelay of com.linkedin.feathr.config.join.TimestampColJoinTimeSettings to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "simulateTimeDelay", value.data()); + _simulateTimeDelayField = value; + } + return this; + } + + @Override + public TimestampColJoinTimeSettings clone() + throws CloneNotSupportedException + { + TimestampColJoinTimeSettings __clone = ((TimestampColJoinTimeSettings) super.clone()); + __clone.__changeListener = new TimestampColJoinTimeSettings.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public TimestampColJoinTimeSettings copy() + throws CloneNotSupportedException + { + TimestampColJoinTimeSettings __copy = ((TimestampColJoinTimeSettings) super.copy()); + __copy._simulateTimeDelayField = null; + __copy._timestampColumnField = null; + __copy.__changeListener = new TimestampColJoinTimeSettings.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final TimestampColJoinTimeSettings __objectRef; + + private ChangeListener(TimestampColJoinTimeSettings reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "simulateTimeDelay": + __objectRef._simulateTimeDelayField = null; + break; + case "timestampColumn": + __objectRef._timestampColumnField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The timestamp column name and timeformat which should be used for joining with the feature data. + * Refer to [[TimestampColumn]]. + * Example, TimestampColumn: { + * def: timestamp + * format: yyyy/MM/dd + * } + * + */ + public com.linkedin.feathr.config.join.TimestampColumn.Fields timestampColumn() { + return new com.linkedin.feathr.config.join.TimestampColumn.Fields(getPathComponents(), "timestampColumn"); + } + + /** + * An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted + * from the input data timestamp while joining with the feature data. + * We do support negative time delays. + * + */ + public com.linkedin.feathr.config.join.TimeOffset.Fields simulateTimeDelay() { + return new com.linkedin.feathr.config.join.TimeOffset.Fields(getPathComponents(), "simulateTimeDelay"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.TimestampColumn.ProjectionMask _timestampColumnMask; + private com.linkedin.feathr.config.join.TimeOffset.ProjectionMask _simulateTimeDelayMask; + + ProjectionMask() { + super(3); + } + + /** + * The timestamp column name and timeformat which should be used for joining with the feature data. + * Refer to [[TimestampColumn]]. + * Example, TimestampColumn: { + * def: timestamp + * format: yyyy/MM/dd + * } + * + */ + public TimestampColJoinTimeSettings.ProjectionMask withTimestampColumn(Function nestedMask) { + _timestampColumnMask = nestedMask.apply(((_timestampColumnMask == null)?TimestampColumn.createMask():_timestampColumnMask)); + getDataMap().put("timestampColumn", _timestampColumnMask.getDataMap()); + return this; + } + + /** + * The timestamp column name and timeformat which should be used for joining with the feature data. + * Refer to [[TimestampColumn]]. + * Example, TimestampColumn: { + * def: timestamp + * format: yyyy/MM/dd + * } + * + */ + public TimestampColJoinTimeSettings.ProjectionMask withTimestampColumn() { + _timestampColumnMask = null; + getDataMap().put("timestampColumn", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted + * from the input data timestamp while joining with the feature data. + * We do support negative time delays. + * + */ + public TimestampColJoinTimeSettings.ProjectionMask withSimulateTimeDelay(Function nestedMask) { + _simulateTimeDelayMask = nestedMask.apply(((_simulateTimeDelayMask == null)?TimeOffset.createMask():_simulateTimeDelayMask)); + getDataMap().put("simulateTimeDelay", _simulateTimeDelayMask.getDataMap()); + return this; + } + + /** + * An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted + * from the input data timestamp while joining with the feature data. + * We do support negative time delays. + * + */ + public TimestampColJoinTimeSettings.ProjectionMask withSimulateTimeDelay() { + _simulateTimeDelayMask = null; + getDataMap().put("simulateTimeDelay", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColumn.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColumn.java new file mode 100644 index 000000000..dae1adacc --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColumn.java @@ -0,0 +1,609 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import java.util.function.Function; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.DataSchema; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.schema.UnionDataSchema; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.RequiredFieldNotPresentException; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.UnionTemplate; + + +/** + * Timestamp column of the input featureiized dataset, which is to be used for the join. + * timestampColumn: { + * def: timestamp + * format: yyyyMMdd + * } + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimestampColumn.pdl.") +public class TimestampColumn + extends RecordTemplate +{ + + private final static TimestampColumn.Fields _fields = new TimestampColumn.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Timestamp column of the input featureiized dataset, which is to be used for the join.\r\ntimestampColumn: {\r\n def: timestamp\r\n format: yyyyMMdd\r\n }*/record TimestampColumn{/**The definiton of the timestamp column, which can be a sql expression involving the timestamp column\r\nor just the column name\r\nExample:- definition: timestamp, timestamp + 10000000.*/definition:union[columnName:string,sparkSqlExpression:/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}]/**Format of the timestamp column. Must confer to java's timestampFormatter or can be\r\nepoch or epoch_millis.\r\nExample:- epoch, epoch_millis, yyyy/MM/dd*/format:/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string}", SchemaFormatType.PDL)); + private TimestampColumn.Definition _definitionField = null; + private String _formatField = null; + private TimestampColumn.ChangeListener __changeListener = new TimestampColumn.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_Definition = SCHEMA.getField("definition"); + private final static RecordDataSchema.Field FIELD_Format = SCHEMA.getField("format"); + + public TimestampColumn() { + super(new DataMap(3, 0.75F), SCHEMA, 2); + addChangeListener(__changeListener); + } + + public TimestampColumn(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static TimestampColumn.Fields fields() { + return _fields; + } + + public static TimestampColumn.ProjectionMask createMask() { + return new TimestampColumn.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for definition + * + * @see TimestampColumn.Fields#definition + */ + public boolean hasDefinition() { + if (_definitionField!= null) { + return true; + } + return super._map.containsKey("definition"); + } + + /** + * Remover for definition + * + * @see TimestampColumn.Fields#definition + */ + public void removeDefinition() { + super._map.remove("definition"); + } + + /** + * Getter for definition + * + * @see TimestampColumn.Fields#definition + */ + public TimestampColumn.Definition getDefinition(GetMode mode) { + switch (mode) { + case STRICT: + return getDefinition(); + case DEFAULT: + case NULL: + if (_definitionField!= null) { + return _definitionField; + } else { + Object __rawValue = super._map.get("definition"); + _definitionField = ((__rawValue == null)?null:new TimestampColumn.Definition(__rawValue)); + return _definitionField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for definition + * + * @return + * Required field. Could be null for partial record. + * @see TimestampColumn.Fields#definition + */ + @Nonnull + public TimestampColumn.Definition getDefinition() { + if (_definitionField!= null) { + return _definitionField; + } else { + Object __rawValue = super._map.get("definition"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("definition"); + } + _definitionField = ((__rawValue == null)?null:new TimestampColumn.Definition(__rawValue)); + return _definitionField; + } + } + + /** + * Setter for definition + * + * @see TimestampColumn.Fields#definition + */ + public TimestampColumn setDefinition(TimestampColumn.Definition value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setDefinition(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field definition of com.linkedin.feathr.config.join.TimestampColumn"); + } else { + CheckedUtil.putWithoutChecking(super._map, "definition", value.data()); + _definitionField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeDefinition(); + } else { + CheckedUtil.putWithoutChecking(super._map, "definition", value.data()); + _definitionField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "definition", value.data()); + _definitionField = value; + } + break; + } + return this; + } + + /** + * Setter for definition + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TimestampColumn.Fields#definition + */ + public TimestampColumn setDefinition( + @Nonnull + TimestampColumn.Definition value) { + if (value == null) { + throw new NullPointerException("Cannot set field definition of com.linkedin.feathr.config.join.TimestampColumn to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "definition", value.data()); + _definitionField = value; + } + return this; + } + + /** + * Existence checker for format + * + * @see TimestampColumn.Fields#format + */ + public boolean hasFormat() { + if (_formatField!= null) { + return true; + } + return super._map.containsKey("format"); + } + + /** + * Remover for format + * + * @see TimestampColumn.Fields#format + */ + public void removeFormat() { + super._map.remove("format"); + } + + /** + * Getter for format + * + * @see TimestampColumn.Fields#format + */ + public String getFormat(GetMode mode) { + switch (mode) { + case STRICT: + return getFormat(); + case DEFAULT: + case NULL: + if (_formatField!= null) { + return _formatField; + } else { + Object __rawValue = super._map.get("format"); + _formatField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _formatField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for format + * + * @return + * Required field. Could be null for partial record. + * @see TimestampColumn.Fields#format + */ + @Nonnull + public String getFormat() { + if (_formatField!= null) { + return _formatField; + } else { + Object __rawValue = super._map.get("format"); + if (__rawValue == null) { + throw new RequiredFieldNotPresentException("format"); + } + _formatField = DataTemplateUtil.coerceStringOutput(__rawValue); + return _formatField; + } + } + + /** + * Setter for format + * + * @see TimestampColumn.Fields#format + */ + public TimestampColumn setFormat(String value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setFormat(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field format of com.linkedin.feathr.config.join.TimestampColumn"); + } else { + CheckedUtil.putWithoutChecking(super._map, "format", value); + _formatField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeFormat(); + } else { + CheckedUtil.putWithoutChecking(super._map, "format", value); + _formatField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "format", value); + _formatField = value; + } + break; + } + return this; + } + + /** + * Setter for format + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see TimestampColumn.Fields#format + */ + public TimestampColumn setFormat( + @Nonnull + String value) { + if (value == null) { + throw new NullPointerException("Cannot set field format of com.linkedin.feathr.config.join.TimestampColumn to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "format", value); + _formatField = value; + } + return this; + } + + @Override + public TimestampColumn clone() + throws CloneNotSupportedException + { + TimestampColumn __clone = ((TimestampColumn) super.clone()); + __clone.__changeListener = new TimestampColumn.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public TimestampColumn copy() + throws CloneNotSupportedException + { + TimestampColumn __copy = ((TimestampColumn) super.copy()); + __copy._formatField = null; + __copy._definitionField = null; + __copy.__changeListener = new TimestampColumn.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final TimestampColumn __objectRef; + + private ChangeListener(TimestampColumn reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "format": + __objectRef._formatField = null; + break; + case "definition": + __objectRef._definitionField = null; + break; + } + } + + } + + @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimestampColumn.pdl.") + public static class Definition + extends UnionTemplate + { + + private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[columnName:string,sparkSqlExpression:{namespace com.linkedin.feathr.config.join/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}}]", SchemaFormatType.PDL)); + private String _columnNameMember = null; + private com.linkedin.feathr.config.join.SparkSqlExpression _sparkSqlExpressionMember = null; + private TimestampColumn.Definition.ChangeListener __changeListener = new TimestampColumn.Definition.ChangeListener(this); + private final static DataSchema MEMBER_ColumnName = SCHEMA.getTypeByMemberKey("columnName"); + private final static DataSchema MEMBER_SparkSqlExpression = SCHEMA.getTypeByMemberKey("sparkSqlExpression"); + + public Definition() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public Definition(Object data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UnionDataSchema dataSchema() { + return SCHEMA; + } + + public static TimestampColumn.Definition createWithColumnName(String value) { + TimestampColumn.Definition newUnion = new TimestampColumn.Definition(); + newUnion.setColumnName(value); + return newUnion; + } + + public boolean isColumnName() { + return memberIs("columnName"); + } + + public String getColumnName() { + checkNotNull(); + if (_columnNameMember!= null) { + return _columnNameMember; + } + Object __rawValue = super._map.get("columnName"); + _columnNameMember = DataTemplateUtil.coerceStringOutput(__rawValue); + return _columnNameMember; + } + + public void setColumnName(String value) { + checkNotNull(); + super._map.clear(); + _columnNameMember = value; + CheckedUtil.putWithoutChecking(super._map, "columnName", value); + } + + public static TimestampColumn.Definition createWithSparkSqlExpression(com.linkedin.feathr.config.join.SparkSqlExpression value) { + TimestampColumn.Definition newUnion = new TimestampColumn.Definition(); + newUnion.setSparkSqlExpression(value); + return newUnion; + } + + public boolean isSparkSqlExpression() { + return memberIs("sparkSqlExpression"); + } + + public com.linkedin.feathr.config.join.SparkSqlExpression getSparkSqlExpression() { + checkNotNull(); + if (_sparkSqlExpressionMember!= null) { + return _sparkSqlExpressionMember; + } + Object __rawValue = super._map.get("sparkSqlExpression"); + _sparkSqlExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.SparkSqlExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); + return _sparkSqlExpressionMember; + } + + public void setSparkSqlExpression(com.linkedin.feathr.config.join.SparkSqlExpression value) { + checkNotNull(); + super._map.clear(); + _sparkSqlExpressionMember = value; + CheckedUtil.putWithoutChecking(super._map, "sparkSqlExpression", value.data()); + } + + public static TimestampColumn.Definition.ProjectionMask createMask() { + return new TimestampColumn.Definition.ProjectionMask(); + } + + @Override + public TimestampColumn.Definition clone() + throws CloneNotSupportedException + { + TimestampColumn.Definition __clone = ((TimestampColumn.Definition) super.clone()); + __clone.__changeListener = new TimestampColumn.Definition.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public TimestampColumn.Definition copy() + throws CloneNotSupportedException + { + TimestampColumn.Definition __copy = ((TimestampColumn.Definition) super.copy()); + __copy._sparkSqlExpressionMember = null; + __copy._columnNameMember = null; + __copy.__changeListener = new TimestampColumn.Definition.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final TimestampColumn.Definition __objectRef; + + private ChangeListener(TimestampColumn.Definition reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "sparkSqlExpression": + __objectRef._sparkSqlExpressionMember = null; + break; + case "columnName": + __objectRef._columnNameMember = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + public PathSpec ColumnName() { + return new PathSpec(getPathComponents(), "columnName"); + } + + public com.linkedin.feathr.config.join.SparkSqlExpression.Fields SparkSqlExpression() { + return new com.linkedin.feathr.config.join.SparkSqlExpression.Fields(getPathComponents(), "sparkSqlExpression"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.SparkSqlExpression.ProjectionMask _SparkSqlExpressionMask; + + ProjectionMask() { + super(3); + } + + public TimestampColumn.Definition.ProjectionMask withColumnName() { + getDataMap().put("columnName", MaskMap.POSITIVE_MASK); + return this; + } + + public TimestampColumn.Definition.ProjectionMask withSparkSqlExpression(Function nestedMask) { + _SparkSqlExpressionMask = nestedMask.apply(((_SparkSqlExpressionMask == null)?com.linkedin.feathr.config.join.SparkSqlExpression.createMask():_SparkSqlExpressionMask)); + getDataMap().put("sparkSqlExpression", _SparkSqlExpressionMask.getDataMap()); + return this; + } + + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * The definiton of the timestamp column, which can be a sql expression involving the timestamp column + * or just the column name + * Example:- definition: timestamp, timestamp + 10000000. + * + */ + public com.linkedin.feathr.config.join.TimestampColumn.Definition.Fields definition() { + return new com.linkedin.feathr.config.join.TimestampColumn.Definition.Fields(getPathComponents(), "definition"); + } + + /** + * Format of the timestamp column. Must confer to java's timestampFormatter or can be + * epoch or epoch_millis. + * Example:- epoch, epoch_millis, yyyy/MM/dd + * + */ + public PathSpec format() { + return new PathSpec(getPathComponents(), "format"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + private com.linkedin.feathr.config.join.TimestampColumn.Definition.ProjectionMask _definitionMask; + + ProjectionMask() { + super(3); + } + + /** + * The definiton of the timestamp column, which can be a sql expression involving the timestamp column + * or just the column name + * Example:- definition: timestamp, timestamp + 10000000. + * + */ + public TimestampColumn.ProjectionMask withDefinition(Function nestedMask) { + _definitionMask = nestedMask.apply(((_definitionMask == null)?TimestampColumn.Definition.createMask():_definitionMask)); + getDataMap().put("definition", _definitionMask.getDataMap()); + return this; + } + + /** + * The definiton of the timestamp column, which can be a sql expression involving the timestamp column + * or just the column name + * Example:- definition: timestamp, timestamp + 10000000. + * + */ + public TimestampColumn.ProjectionMask withDefinition() { + _definitionMask = null; + getDataMap().put("definition", MaskMap.POSITIVE_MASK); + return this; + } + + /** + * Format of the timestamp column. Must confer to java's timestampFormatter or can be + * epoch or epoch_millis. + * Example:- epoch, epoch_millis, yyyy/MM/dd + * + */ + public TimestampColumn.ProjectionMask withFormat() { + getDataMap().put("format", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/UseLatestJoinTimeSettings.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/UseLatestJoinTimeSettings.java new file mode 100644 index 000000000..7e796b319 --- /dev/null +++ b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/UseLatestJoinTimeSettings.java @@ -0,0 +1,280 @@ + +package com.linkedin.feathr.config.join; + +import java.util.List; +import javax.annotation.Generated; +import javax.annotation.Nonnull; +import com.linkedin.data.DataMap; +import com.linkedin.data.collections.CheckedUtil; +import com.linkedin.data.schema.MaskMap; +import com.linkedin.data.schema.PathSpec; +import com.linkedin.data.schema.RecordDataSchema; +import com.linkedin.data.schema.SchemaFormatType; +import com.linkedin.data.template.DataTemplateUtil; +import com.linkedin.data.template.GetMode; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.data.template.SetMode; + + +/** + * Settings needed when the input data is to be joined with the latest available feature data. + * joinTimeSettings: { + * useLatestFeatureData: true + * } + * + */ +@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\UseLatestJoinTimeSettings.pdl.") +public class UseLatestJoinTimeSettings + extends RecordTemplate +{ + + private final static UseLatestJoinTimeSettings.Fields _fields = new UseLatestJoinTimeSettings.Fields(); + private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Settings needed when the input data is to be joined with the latest available feature data.\r\njoinTimeSettings: {\r\n useLatestFeatureData: true\r\n}*/record UseLatestJoinTimeSettings{/**Boolean value, if set to true, indicates that the latest available feature data is to be used for joining.\r\nWhen useLatestFeatureData is set, there should be no other time-based parameters.*/useLatestFeatureData:boolean=true}", SchemaFormatType.PDL)); + private Boolean _useLatestFeatureDataField = null; + private UseLatestJoinTimeSettings.ChangeListener __changeListener = new UseLatestJoinTimeSettings.ChangeListener(this); + private final static RecordDataSchema.Field FIELD_UseLatestFeatureData = SCHEMA.getField("useLatestFeatureData"); + private final static Boolean DEFAULT_UseLatestFeatureData; + + static { + DEFAULT_UseLatestFeatureData = DataTemplateUtil.coerceBooleanOutput(FIELD_UseLatestFeatureData.getDefault()); + } + + public UseLatestJoinTimeSettings() { + super(new DataMap(2, 0.75F), SCHEMA); + addChangeListener(__changeListener); + } + + public UseLatestJoinTimeSettings(DataMap data) { + super(data, SCHEMA); + addChangeListener(__changeListener); + } + + public static UseLatestJoinTimeSettings.Fields fields() { + return _fields; + } + + public static UseLatestJoinTimeSettings.ProjectionMask createMask() { + return new UseLatestJoinTimeSettings.ProjectionMask(); + } + + public static RecordDataSchema dataSchema() { + return SCHEMA; + } + + /** + * Existence checker for useLatestFeatureData + * + * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData + */ + public boolean hasUseLatestFeatureData() { + if (_useLatestFeatureDataField!= null) { + return true; + } + return super._map.containsKey("useLatestFeatureData"); + } + + /** + * Remover for useLatestFeatureData + * + * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData + */ + public void removeUseLatestFeatureData() { + super._map.remove("useLatestFeatureData"); + } + + /** + * Getter for useLatestFeatureData + * + * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData + */ + public Boolean isUseLatestFeatureData(GetMode mode) { + switch (mode) { + case STRICT: + case DEFAULT: + return isUseLatestFeatureData(); + case NULL: + if (_useLatestFeatureDataField!= null) { + return _useLatestFeatureDataField; + } else { + Object __rawValue = super._map.get("useLatestFeatureData"); + _useLatestFeatureDataField = DataTemplateUtil.coerceBooleanOutput(__rawValue); + return _useLatestFeatureDataField; + } + } + throw new IllegalStateException(("Unknown mode "+ mode)); + } + + /** + * Getter for useLatestFeatureData + * + * @return + * Required field. Could be null for partial record. + * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData + */ + @Nonnull + public Boolean isUseLatestFeatureData() { + if (_useLatestFeatureDataField!= null) { + return _useLatestFeatureDataField; + } else { + Object __rawValue = super._map.get("useLatestFeatureData"); + if (__rawValue == null) { + return DEFAULT_UseLatestFeatureData; + } + _useLatestFeatureDataField = DataTemplateUtil.coerceBooleanOutput(__rawValue); + return _useLatestFeatureDataField; + } + } + + /** + * Setter for useLatestFeatureData + * + * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData + */ + public UseLatestJoinTimeSettings setUseLatestFeatureData(Boolean value, SetMode mode) { + switch (mode) { + case DISALLOW_NULL: + return setUseLatestFeatureData(value); + case REMOVE_OPTIONAL_IF_NULL: + if (value == null) { + throw new IllegalArgumentException("Cannot remove mandatory field useLatestFeatureData of com.linkedin.feathr.config.join.UseLatestJoinTimeSettings"); + } else { + CheckedUtil.putWithoutChecking(super._map, "useLatestFeatureData", value); + _useLatestFeatureDataField = value; + } + break; + case REMOVE_IF_NULL: + if (value == null) { + removeUseLatestFeatureData(); + } else { + CheckedUtil.putWithoutChecking(super._map, "useLatestFeatureData", value); + _useLatestFeatureDataField = value; + } + break; + case IGNORE_NULL: + if (value!= null) { + CheckedUtil.putWithoutChecking(super._map, "useLatestFeatureData", value); + _useLatestFeatureDataField = value; + } + break; + } + return this; + } + + /** + * Setter for useLatestFeatureData + * + * @param value + * Must not be null. For more control, use setters with mode instead. + * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData + */ + public UseLatestJoinTimeSettings setUseLatestFeatureData( + @Nonnull + Boolean value) { + if (value == null) { + throw new NullPointerException("Cannot set field useLatestFeatureData of com.linkedin.feathr.config.join.UseLatestJoinTimeSettings to null"); + } else { + CheckedUtil.putWithoutChecking(super._map, "useLatestFeatureData", value); + _useLatestFeatureDataField = value; + } + return this; + } + + /** + * Setter for useLatestFeatureData + * + * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData + */ + public UseLatestJoinTimeSettings setUseLatestFeatureData(boolean value) { + CheckedUtil.putWithoutChecking(super._map, "useLatestFeatureData", value); + _useLatestFeatureDataField = value; + return this; + } + + @Override + public UseLatestJoinTimeSettings clone() + throws CloneNotSupportedException + { + UseLatestJoinTimeSettings __clone = ((UseLatestJoinTimeSettings) super.clone()); + __clone.__changeListener = new UseLatestJoinTimeSettings.ChangeListener(__clone); + __clone.addChangeListener(__clone.__changeListener); + return __clone; + } + + @Override + public UseLatestJoinTimeSettings copy() + throws CloneNotSupportedException + { + UseLatestJoinTimeSettings __copy = ((UseLatestJoinTimeSettings) super.copy()); + __copy._useLatestFeatureDataField = null; + __copy.__changeListener = new UseLatestJoinTimeSettings.ChangeListener(__copy); + __copy.addChangeListener(__copy.__changeListener); + return __copy; + } + + private static class ChangeListener + implements com.linkedin.data.collections.CheckedMap.ChangeListener + { + + private final UseLatestJoinTimeSettings __objectRef; + + private ChangeListener(UseLatestJoinTimeSettings reference) { + __objectRef = reference; + } + + @Override + public void onUnderlyingMapChanged(String key, Object value) { + switch (key) { + case "useLatestFeatureData": + __objectRef._useLatestFeatureDataField = null; + break; + } + } + + } + + public static class Fields + extends PathSpec + { + + + public Fields(List path, String name) { + super(path, name); + } + + public Fields() { + super(); + } + + /** + * Boolean value, if set to true, indicates that the latest available feature data is to be used for joining. + * When useLatestFeatureData is set, there should be no other time-based parameters. + * + */ + public PathSpec useLatestFeatureData() { + return new PathSpec(getPathComponents(), "useLatestFeatureData"); + } + + } + + public static class ProjectionMask + extends MaskMap + { + + + ProjectionMask() { + super(2); + } + + /** + * Boolean value, if set to true, indicates that the latest available feature data is to be used for joining. + * When useLatestFeatureData is set, there should be no other time-based parameters. + * + */ + public UseLatestJoinTimeSettings.ProjectionMask withUseLatestFeatureData() { + getDataMap().put("useLatestFeatureData", MaskMap.POSITIVE_MASK); + return this; + } + + } + +} From 0617f416b225b46b2b70b10c22d57b40d64fcd6d Mon Sep 17 00:00:00 2001 From: Xiaoyong Zhu Date: Tue, 17 Jan 2023 00:23:54 -0800 Subject: [PATCH 17/17] Revert "udpate branch" This reverts commit 6c2a415cb4ff34c55e099fc3633b52496d1a59c9. --- .../linkedin/feathr/compute/AbstractNode.java | 415 ----- .../linkedin/feathr/compute/Aggregation.java | 1016 ----------- .../feathr/compute/AggregationFunction.java | 394 ---- .../feathr/compute/AggregationType.java | 78 - .../com/linkedin/feathr/compute/AnyNode.java | 357 ---- .../linkedin/feathr/compute/AnyNodeArray.java | 117 -- .../linkedin/feathr/compute/ComputeGraph.java | 464 ----- .../linkedin/feathr/compute/ConcreteKey.java | 300 ---- .../linkedin/feathr/compute/DataSource.java | 1232 ------------- .../feathr/compute/DataSourceType.java | 43 - .../feathr/compute/DateTimeInterval.java | 377 ---- .../linkedin/feathr/compute/Dimension.java | 390 ---- .../feathr/compute/DimensionArray.java | 118 -- .../feathr/compute/DimensionType.java | 48 - .../com/linkedin/feathr/compute/External.java | 557 ------ .../linkedin/feathr/compute/FeatureValue.java | 431 ----- .../feathr/compute/FeatureVersion.java | 526 ------ .../feathr/compute/FrameFeatureType.java | 72 - .../feathr/compute/KeyExpressionType.java | 44 - .../linkedin/feathr/compute/KeyReference.java | 272 --- .../feathr/compute/KeyReferenceArray.java | 118 -- .../linkedin/feathr/compute/LateralView.java | 553 ------ .../feathr/compute/LateralViewArray.java | 118 -- .../com/linkedin/feathr/compute/Lookup.java | 1587 ----------------- .../feathr/compute/MvelExpression.java | 260 --- .../com/linkedin/feathr/compute/NodeId.java | 30 - .../feathr/compute/NodeReference.java | 488 ----- .../feathr/compute/NodeReferenceArray.java | 118 -- .../feathr/compute/OfflineKeyFunction.java | 477 ----- .../linkedin/feathr/compute/OperatorId.java | 30 - .../feathr/compute/SlidingWindowFeature.java | 1574 ---------------- .../feathr/compute/SqlExpression.java | 260 --- .../feathr/compute/TensorCategory.java | 42 - .../feathr/compute/TensorFeatureFormat.java | 603 ------- .../com/linkedin/feathr/compute/Time.java | 30 - .../linkedin/feathr/compute/TimestampCol.java | 401 ----- .../feathr/compute/Transformation.java | 1065 ----------- .../compute/TransformationFunction.java | 384 ---- .../com/linkedin/feathr/compute/Unit.java | 48 - .../feathr/compute/UserDefinedFunction.java | 407 ----- .../linkedin/feathr/compute/ValueType.java | 66 - .../com/linkedin/feathr/compute/Window.java | 412 ----- .../feathr/config/join/AbsoluteDateRange.java | 432 ----- .../feathr/config/join/AbsoluteTimeRange.java | 802 --------- .../com/linkedin/feathr/config/join/Date.java | 575 ------ .../config/join/FrameFeatureJoinConfig.java | 520 ------ .../linkedin/feathr/config/join/HourTime.java | 727 -------- .../config/join/InputDataTimeSettings.java | 502 ------ .../feathr/config/join/JoinTimeSettings.java | 231 --- .../feathr/config/join/JoiningFeature.java | 1136 ------------ .../config/join/JoiningFeatureArray.java | 118 -- .../feathr/config/join/RelativeDateRange.java | 443 ----- .../feathr/config/join/RelativeTimeRange.java | 453 ----- .../linkedin/feathr/config/join/Settings.java | 400 ----- .../config/join/SparkSqlExpression.java | 260 --- .../feathr/config/join/TimeFormat.java | 31 - .../feathr/config/join/TimeOffset.java | 414 ----- .../linkedin/feathr/config/join/TimeUnit.java | 48 - .../feathr/config/join/TimeWindow.java | 412 ----- .../join/TimestampColJoinTimeSettings.java | 432 ----- .../feathr/config/join/TimestampColumn.java | 609 ------- .../join/UseLatestJoinTimeSettings.java | 280 --- 62 files changed, 25147 deletions(-) delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AbstractNode.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Aggregation.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationFunction.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationType.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNode.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNodeArray.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ComputeGraph.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ConcreteKey.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSource.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSourceType.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DateTimeInterval.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Dimension.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionArray.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionType.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/External.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureValue.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureVersion.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FrameFeatureType.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyExpressionType.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReference.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReferenceArray.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralView.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralViewArray.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Lookup.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/MvelExpression.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeId.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReference.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReferenceArray.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OfflineKeyFunction.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OperatorId.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SlidingWindowFeature.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SqlExpression.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorCategory.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorFeatureFormat.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Time.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TimestampCol.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Transformation.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TransformationFunction.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Unit.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/UserDefinedFunction.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ValueType.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Window.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteDateRange.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteTimeRange.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Date.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/FrameFeatureJoinConfig.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/HourTime.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/InputDataTimeSettings.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoinTimeSettings.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeature.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeatureArray.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeDateRange.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeTimeRange.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Settings.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/SparkSqlExpression.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeFormat.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeOffset.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeUnit.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeWindow.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColJoinTimeSettings.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColumn.java delete mode 100644 feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/UseLatestJoinTimeSettings.java diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AbstractNode.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AbstractNode.java deleted file mode 100644 index 00f947067..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AbstractNode.java +++ /dev/null @@ -1,415 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * Generic abstraction of a node. All other nodes should derive from this node. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\AbstractNode.pdl.") -public class AbstractNode - extends RecordTemplate -{ - - private final static AbstractNode.Fields _fields = new AbstractNode.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}", SchemaFormatType.PDL)); - private Integer _idField = null; - private ConcreteKey _concreteKeyField = null; - private AbstractNode.ChangeListener __changeListener = new AbstractNode.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); - private final static RecordDataSchema.Field FIELD_ConcreteKey = SCHEMA.getField("concreteKey"); - - public AbstractNode() { - super(new DataMap(3, 0.75F), SCHEMA, 2); - addChangeListener(__changeListener); - } - - public AbstractNode(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static AbstractNode.Fields fields() { - return _fields; - } - - public static AbstractNode.ProjectionMask createMask() { - return new AbstractNode.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for id - * - * @see AbstractNode.Fields#id - */ - public boolean hasId() { - if (_idField!= null) { - return true; - } - return super._map.containsKey("id"); - } - - /** - * Remover for id - * - * @see AbstractNode.Fields#id - */ - public void removeId() { - super._map.remove("id"); - } - - /** - * Getter for id - * - * @see AbstractNode.Fields#id - */ - public Integer getId(GetMode mode) { - switch (mode) { - case STRICT: - return getId(); - case DEFAULT: - case NULL: - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for id - * - * @return - * Required field. Could be null for partial record. - * @see AbstractNode.Fields#id - */ - @Nonnull - public Integer getId() { - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("id"); - } - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - - /** - * Setter for id - * - * @see AbstractNode.Fields#id - */ - public AbstractNode setId(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setId(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.AbstractNode"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeId(); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - } - return this; - } - - /** - * Setter for id - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see AbstractNode.Fields#id - */ - public AbstractNode setId( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.AbstractNode to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - return this; - } - - /** - * Setter for id - * - * @see AbstractNode.Fields#id - */ - public AbstractNode setId(int value) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - return this; - } - - /** - * Existence checker for concreteKey - * - * @see AbstractNode.Fields#concreteKey - */ - public boolean hasConcreteKey() { - if (_concreteKeyField!= null) { - return true; - } - return super._map.containsKey("concreteKey"); - } - - /** - * Remover for concreteKey - * - * @see AbstractNode.Fields#concreteKey - */ - public void removeConcreteKey() { - super._map.remove("concreteKey"); - } - - /** - * Getter for concreteKey - * - * @see AbstractNode.Fields#concreteKey - */ - public ConcreteKey getConcreteKey(GetMode mode) { - return getConcreteKey(); - } - - /** - * Getter for concreteKey - * - * @return - * Optional field. Always check for null. - * @see AbstractNode.Fields#concreteKey - */ - @Nullable - public ConcreteKey getConcreteKey() { - if (_concreteKeyField!= null) { - return _concreteKeyField; - } else { - Object __rawValue = super._map.get("concreteKey"); - _concreteKeyField = ((__rawValue == null)?null:new ConcreteKey(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _concreteKeyField; - } - } - - /** - * Setter for concreteKey - * - * @see AbstractNode.Fields#concreteKey - */ - public AbstractNode setConcreteKey(ConcreteKey value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setConcreteKey(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeConcreteKey(); - } else { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - break; - } - return this; - } - - /** - * Setter for concreteKey - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see AbstractNode.Fields#concreteKey - */ - public AbstractNode setConcreteKey( - @Nonnull - ConcreteKey value) { - if (value == null) { - throw new NullPointerException("Cannot set field concreteKey of com.linkedin.feathr.compute.AbstractNode to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - return this; - } - - @Override - public AbstractNode clone() - throws CloneNotSupportedException - { - AbstractNode __clone = ((AbstractNode) super.clone()); - __clone.__changeListener = new AbstractNode.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public AbstractNode copy() - throws CloneNotSupportedException - { - AbstractNode __copy = ((AbstractNode) super.copy()); - __copy._idField = null; - __copy._concreteKeyField = null; - __copy.__changeListener = new AbstractNode.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final AbstractNode __objectRef; - - private ChangeListener(AbstractNode reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "id": - __objectRef._idField = null; - break; - case "concreteKey": - __objectRef._concreteKeyField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The node would be represented by this id. - * - */ - public PathSpec id() { - return new PathSpec(getPathComponents(), "id"); - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public com.linkedin.feathr.compute.ConcreteKey.Fields concreteKey() { - return new com.linkedin.feathr.compute.ConcreteKey.Fields(getPathComponents(), "concreteKey"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.ConcreteKey.ProjectionMask _concreteKeyMask; - - ProjectionMask() { - super(3); - } - - /** - * The node would be represented by this id. - * - */ - public AbstractNode.ProjectionMask withId() { - getDataMap().put("id", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public AbstractNode.ProjectionMask withConcreteKey(Function nestedMask) { - _concreteKeyMask = nestedMask.apply(((_concreteKeyMask == null)?ConcreteKey.createMask():_concreteKeyMask)); - getDataMap().put("concreteKey", _concreteKeyMask.getDataMap()); - return this; - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public AbstractNode.ProjectionMask withConcreteKey() { - _concreteKeyMask = null; - getDataMap().put("concreteKey", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Aggregation.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Aggregation.java deleted file mode 100644 index 934ca5be8..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Aggregation.java +++ /dev/null @@ -1,1016 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * A node to represent an aggregation step. The aggregation inputs like the groupBy field, agg function are delegated to [[AggregationFunction]]. - * This node can represent a feature. As of now, in this step we will be using the SWA library from Spark-algorithms. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Aggregation.pdl.") -public class Aggregation - extends RecordTemplate -{ - - private final static Aggregation.Fields _fields = new Aggregation.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**A node to represent an aggregation step. The aggregation inputs like the groupBy field, agg function are delegated to [[AggregationFunction]].\r\nThis node can represent a feature. As of now, in this step we will be using the SWA library from Spark-algorithms.*/record Aggregation includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node.*/input:/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}/**All the aggregation related parameters and functions are bundled into this.*/function:/**All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node.*/record AggregationFunction{/**The aggregation function.*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**All the aggregation parameters should be bundled into this map. For now, the possible parameters are:-\r\na. target_column - Aggregation column\r\nb. window_size - aggregation window size\r\nc. window unit - aggregation window unit (ex - day, hour)\r\nd. lateral_view_expression - definition of a lateral view for the feature.\r\ne. lateral_view_table_alias - An alias for the lateral view\r\nf. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression.\r\ng. groupBy - groupBy columns. Should be a sparkSql expression.*/parameters:optional map[string,string]}/**If the node is representing a feature, the feature name should be associated with the node.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}", SchemaFormatType.PDL)); - private Integer _idField = null; - private ConcreteKey _concreteKeyField = null; - private NodeReference _inputField = null; - private AggregationFunction _functionField = null; - private String _featureNameField = null; - private FeatureVersion _featureVersionField = null; - private Aggregation.ChangeListener __changeListener = new Aggregation.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); - private final static RecordDataSchema.Field FIELD_ConcreteKey = SCHEMA.getField("concreteKey"); - private final static RecordDataSchema.Field FIELD_Input = SCHEMA.getField("input"); - private final static RecordDataSchema.Field FIELD_Function = SCHEMA.getField("function"); - private final static RecordDataSchema.Field FIELD_FeatureName = SCHEMA.getField("featureName"); - private final static RecordDataSchema.Field FIELD_FeatureVersion = SCHEMA.getField("featureVersion"); - - public Aggregation() { - super(new DataMap(8, 0.75F), SCHEMA, 6); - addChangeListener(__changeListener); - } - - public Aggregation(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static Aggregation.Fields fields() { - return _fields; - } - - public static Aggregation.ProjectionMask createMask() { - return new Aggregation.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for id - * - * @see Aggregation.Fields#id - */ - public boolean hasId() { - if (_idField!= null) { - return true; - } - return super._map.containsKey("id"); - } - - /** - * Remover for id - * - * @see Aggregation.Fields#id - */ - public void removeId() { - super._map.remove("id"); - } - - /** - * Getter for id - * - * @see Aggregation.Fields#id - */ - public Integer getId(GetMode mode) { - switch (mode) { - case STRICT: - return getId(); - case DEFAULT: - case NULL: - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for id - * - * @return - * Required field. Could be null for partial record. - * @see Aggregation.Fields#id - */ - @Nonnull - public Integer getId() { - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("id"); - } - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - - /** - * Setter for id - * - * @see Aggregation.Fields#id - */ - public Aggregation setId(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setId(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.Aggregation"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeId(); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - } - return this; - } - - /** - * Setter for id - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Aggregation.Fields#id - */ - public Aggregation setId( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.Aggregation to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - return this; - } - - /** - * Setter for id - * - * @see Aggregation.Fields#id - */ - public Aggregation setId(int value) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - return this; - } - - /** - * Existence checker for concreteKey - * - * @see Aggregation.Fields#concreteKey - */ - public boolean hasConcreteKey() { - if (_concreteKeyField!= null) { - return true; - } - return super._map.containsKey("concreteKey"); - } - - /** - * Remover for concreteKey - * - * @see Aggregation.Fields#concreteKey - */ - public void removeConcreteKey() { - super._map.remove("concreteKey"); - } - - /** - * Getter for concreteKey - * - * @see Aggregation.Fields#concreteKey - */ - public ConcreteKey getConcreteKey(GetMode mode) { - return getConcreteKey(); - } - - /** - * Getter for concreteKey - * - * @return - * Optional field. Always check for null. - * @see Aggregation.Fields#concreteKey - */ - @Nullable - public ConcreteKey getConcreteKey() { - if (_concreteKeyField!= null) { - return _concreteKeyField; - } else { - Object __rawValue = super._map.get("concreteKey"); - _concreteKeyField = ((__rawValue == null)?null:new ConcreteKey(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _concreteKeyField; - } - } - - /** - * Setter for concreteKey - * - * @see Aggregation.Fields#concreteKey - */ - public Aggregation setConcreteKey(ConcreteKey value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setConcreteKey(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeConcreteKey(); - } else { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - break; - } - return this; - } - - /** - * Setter for concreteKey - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Aggregation.Fields#concreteKey - */ - public Aggregation setConcreteKey( - @Nonnull - ConcreteKey value) { - if (value == null) { - throw new NullPointerException("Cannot set field concreteKey of com.linkedin.feathr.compute.Aggregation to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - return this; - } - - /** - * Existence checker for input - * - * @see Aggregation.Fields#input - */ - public boolean hasInput() { - if (_inputField!= null) { - return true; - } - return super._map.containsKey("input"); - } - - /** - * Remover for input - * - * @see Aggregation.Fields#input - */ - public void removeInput() { - super._map.remove("input"); - } - - /** - * Getter for input - * - * @see Aggregation.Fields#input - */ - public NodeReference getInput(GetMode mode) { - switch (mode) { - case STRICT: - return getInput(); - case DEFAULT: - case NULL: - if (_inputField!= null) { - return _inputField; - } else { - Object __rawValue = super._map.get("input"); - _inputField = ((__rawValue == null)?null:new NodeReference(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _inputField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for input - * - * @return - * Required field. Could be null for partial record. - * @see Aggregation.Fields#input - */ - @Nonnull - public NodeReference getInput() { - if (_inputField!= null) { - return _inputField; - } else { - Object __rawValue = super._map.get("input"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("input"); - } - _inputField = ((__rawValue == null)?null:new NodeReference(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _inputField; - } - } - - /** - * Setter for input - * - * @see Aggregation.Fields#input - */ - public Aggregation setInput(NodeReference value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setInput(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field input of com.linkedin.feathr.compute.Aggregation"); - } else { - CheckedUtil.putWithoutChecking(super._map, "input", value.data()); - _inputField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeInput(); - } else { - CheckedUtil.putWithoutChecking(super._map, "input", value.data()); - _inputField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "input", value.data()); - _inputField = value; - } - break; - } - return this; - } - - /** - * Setter for input - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Aggregation.Fields#input - */ - public Aggregation setInput( - @Nonnull - NodeReference value) { - if (value == null) { - throw new NullPointerException("Cannot set field input of com.linkedin.feathr.compute.Aggregation to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "input", value.data()); - _inputField = value; - } - return this; - } - - /** - * Existence checker for function - * - * @see Aggregation.Fields#function - */ - public boolean hasFunction() { - if (_functionField!= null) { - return true; - } - return super._map.containsKey("function"); - } - - /** - * Remover for function - * - * @see Aggregation.Fields#function - */ - public void removeFunction() { - super._map.remove("function"); - } - - /** - * Getter for function - * - * @see Aggregation.Fields#function - */ - public AggregationFunction getFunction(GetMode mode) { - switch (mode) { - case STRICT: - return getFunction(); - case DEFAULT: - case NULL: - if (_functionField!= null) { - return _functionField; - } else { - Object __rawValue = super._map.get("function"); - _functionField = ((__rawValue == null)?null:new AggregationFunction(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _functionField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for function - * - * @return - * Required field. Could be null for partial record. - * @see Aggregation.Fields#function - */ - @Nonnull - public AggregationFunction getFunction() { - if (_functionField!= null) { - return _functionField; - } else { - Object __rawValue = super._map.get("function"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("function"); - } - _functionField = ((__rawValue == null)?null:new AggregationFunction(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _functionField; - } - } - - /** - * Setter for function - * - * @see Aggregation.Fields#function - */ - public Aggregation setFunction(AggregationFunction value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFunction(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field function of com.linkedin.feathr.compute.Aggregation"); - } else { - CheckedUtil.putWithoutChecking(super._map, "function", value.data()); - _functionField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeFunction(); - } else { - CheckedUtil.putWithoutChecking(super._map, "function", value.data()); - _functionField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "function", value.data()); - _functionField = value; - } - break; - } - return this; - } - - /** - * Setter for function - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Aggregation.Fields#function - */ - public Aggregation setFunction( - @Nonnull - AggregationFunction value) { - if (value == null) { - throw new NullPointerException("Cannot set field function of com.linkedin.feathr.compute.Aggregation to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "function", value.data()); - _functionField = value; - } - return this; - } - - /** - * Existence checker for featureName - * - * @see Aggregation.Fields#featureName - */ - public boolean hasFeatureName() { - if (_featureNameField!= null) { - return true; - } - return super._map.containsKey("featureName"); - } - - /** - * Remover for featureName - * - * @see Aggregation.Fields#featureName - */ - public void removeFeatureName() { - super._map.remove("featureName"); - } - - /** - * Getter for featureName - * - * @see Aggregation.Fields#featureName - */ - public String getFeatureName(GetMode mode) { - switch (mode) { - case STRICT: - return getFeatureName(); - case DEFAULT: - case NULL: - if (_featureNameField!= null) { - return _featureNameField; - } else { - Object __rawValue = super._map.get("featureName"); - _featureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _featureNameField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for featureName - * - * @return - * Required field. Could be null for partial record. - * @see Aggregation.Fields#featureName - */ - @Nonnull - public String getFeatureName() { - if (_featureNameField!= null) { - return _featureNameField; - } else { - Object __rawValue = super._map.get("featureName"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("featureName"); - } - _featureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _featureNameField; - } - } - - /** - * Setter for featureName - * - * @see Aggregation.Fields#featureName - */ - public Aggregation setFeatureName(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFeatureName(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field featureName of com.linkedin.feathr.compute.Aggregation"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureName", value); - _featureNameField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeFeatureName(); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureName", value); - _featureNameField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "featureName", value); - _featureNameField = value; - } - break; - } - return this; - } - - /** - * Setter for featureName - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Aggregation.Fields#featureName - */ - public Aggregation setFeatureName( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field featureName of com.linkedin.feathr.compute.Aggregation to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureName", value); - _featureNameField = value; - } - return this; - } - - /** - * Existence checker for featureVersion - * - * @see Aggregation.Fields#featureVersion - */ - public boolean hasFeatureVersion() { - if (_featureVersionField!= null) { - return true; - } - return super._map.containsKey("featureVersion"); - } - - /** - * Remover for featureVersion - * - * @see Aggregation.Fields#featureVersion - */ - public void removeFeatureVersion() { - super._map.remove("featureVersion"); - } - - /** - * Getter for featureVersion - * - * @see Aggregation.Fields#featureVersion - */ - public FeatureVersion getFeatureVersion(GetMode mode) { - switch (mode) { - case STRICT: - return getFeatureVersion(); - case DEFAULT: - case NULL: - if (_featureVersionField!= null) { - return _featureVersionField; - } else { - Object __rawValue = super._map.get("featureVersion"); - _featureVersionField = ((__rawValue == null)?null:new FeatureVersion(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _featureVersionField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for featureVersion - * - * @return - * Required field. Could be null for partial record. - * @see Aggregation.Fields#featureVersion - */ - @Nonnull - public FeatureVersion getFeatureVersion() { - if (_featureVersionField!= null) { - return _featureVersionField; - } else { - Object __rawValue = super._map.get("featureVersion"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("featureVersion"); - } - _featureVersionField = ((__rawValue == null)?null:new FeatureVersion(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _featureVersionField; - } - } - - /** - * Setter for featureVersion - * - * @see Aggregation.Fields#featureVersion - */ - public Aggregation setFeatureVersion(FeatureVersion value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFeatureVersion(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field featureVersion of com.linkedin.feathr.compute.Aggregation"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); - _featureVersionField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeFeatureVersion(); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); - _featureVersionField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); - _featureVersionField = value; - } - break; - } - return this; - } - - /** - * Setter for featureVersion - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Aggregation.Fields#featureVersion - */ - public Aggregation setFeatureVersion( - @Nonnull - FeatureVersion value) { - if (value == null) { - throw new NullPointerException("Cannot set field featureVersion of com.linkedin.feathr.compute.Aggregation to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); - _featureVersionField = value; - } - return this; - } - - @Override - public Aggregation clone() - throws CloneNotSupportedException - { - Aggregation __clone = ((Aggregation) super.clone()); - __clone.__changeListener = new Aggregation.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public Aggregation copy() - throws CloneNotSupportedException - { - Aggregation __copy = ((Aggregation) super.copy()); - __copy._inputField = null; - __copy._featureNameField = null; - __copy._functionField = null; - __copy._idField = null; - __copy._concreteKeyField = null; - __copy._featureVersionField = null; - __copy.__changeListener = new Aggregation.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final Aggregation __objectRef; - - private ChangeListener(Aggregation reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "input": - __objectRef._inputField = null; - break; - case "featureName": - __objectRef._featureNameField = null; - break; - case "function": - __objectRef._functionField = null; - break; - case "id": - __objectRef._idField = null; - break; - case "concreteKey": - __objectRef._concreteKeyField = null; - break; - case "featureVersion": - __objectRef._featureVersionField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The node would be represented by this id. - * - */ - public PathSpec id() { - return new PathSpec(getPathComponents(), "id"); - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public com.linkedin.feathr.compute.ConcreteKey.Fields concreteKey() { - return new com.linkedin.feathr.compute.ConcreteKey.Fields(getPathComponents(), "concreteKey"); - } - - /** - * The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node. - * - */ - public com.linkedin.feathr.compute.NodeReference.Fields input() { - return new com.linkedin.feathr.compute.NodeReference.Fields(getPathComponents(), "input"); - } - - /** - * All the aggregation related parameters and functions are bundled into this. - * - */ - public com.linkedin.feathr.compute.AggregationFunction.Fields function() { - return new com.linkedin.feathr.compute.AggregationFunction.Fields(getPathComponents(), "function"); - } - - /** - * If the node is representing a feature, the feature name should be associated with the node. - * - */ - public PathSpec featureName() { - return new PathSpec(getPathComponents(), "featureName"); - } - - /** - * feature version of the feature - * - */ - public com.linkedin.feathr.compute.FeatureVersion.Fields featureVersion() { - return new com.linkedin.feathr.compute.FeatureVersion.Fields(getPathComponents(), "featureVersion"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.ConcreteKey.ProjectionMask _concreteKeyMask; - private com.linkedin.feathr.compute.NodeReference.ProjectionMask _inputMask; - private com.linkedin.feathr.compute.AggregationFunction.ProjectionMask _functionMask; - private com.linkedin.feathr.compute.FeatureVersion.ProjectionMask _featureVersionMask; - - ProjectionMask() { - super(8); - } - - /** - * The node would be represented by this id. - * - */ - public Aggregation.ProjectionMask withId() { - getDataMap().put("id", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public Aggregation.ProjectionMask withConcreteKey(Function nestedMask) { - _concreteKeyMask = nestedMask.apply(((_concreteKeyMask == null)?ConcreteKey.createMask():_concreteKeyMask)); - getDataMap().put("concreteKey", _concreteKeyMask.getDataMap()); - return this; - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public Aggregation.ProjectionMask withConcreteKey() { - _concreteKeyMask = null; - getDataMap().put("concreteKey", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node. - * - */ - public Aggregation.ProjectionMask withInput(Function nestedMask) { - _inputMask = nestedMask.apply(((_inputMask == null)?NodeReference.createMask():_inputMask)); - getDataMap().put("input", _inputMask.getDataMap()); - return this; - } - - /** - * The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node. - * - */ - public Aggregation.ProjectionMask withInput() { - _inputMask = null; - getDataMap().put("input", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * All the aggregation related parameters and functions are bundled into this. - * - */ - public Aggregation.ProjectionMask withFunction(Function nestedMask) { - _functionMask = nestedMask.apply(((_functionMask == null)?AggregationFunction.createMask():_functionMask)); - getDataMap().put("function", _functionMask.getDataMap()); - return this; - } - - /** - * All the aggregation related parameters and functions are bundled into this. - * - */ - public Aggregation.ProjectionMask withFunction() { - _functionMask = null; - getDataMap().put("function", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * If the node is representing a feature, the feature name should be associated with the node. - * - */ - public Aggregation.ProjectionMask withFeatureName() { - getDataMap().put("featureName", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * feature version of the feature - * - */ - public Aggregation.ProjectionMask withFeatureVersion(Function nestedMask) { - _featureVersionMask = nestedMask.apply(((_featureVersionMask == null)?FeatureVersion.createMask():_featureVersionMask)); - getDataMap().put("featureVersion", _featureVersionMask.getDataMap()); - return this; - } - - /** - * feature version of the feature - * - */ - public Aggregation.ProjectionMask withFeatureVersion() { - _featureVersionMask = null; - getDataMap().put("featureVersion", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationFunction.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationFunction.java deleted file mode 100644 index ab0933539..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationFunction.java +++ /dev/null @@ -1,394 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; -import com.linkedin.data.template.StringMap; - - -/** - * All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\AggregationFunction.pdl.") -public class AggregationFunction - extends RecordTemplate -{ - - private final static AggregationFunction.Fields _fields = new AggregationFunction.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node.*/record AggregationFunction{/**The aggregation function.*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**All the aggregation parameters should be bundled into this map. For now, the possible parameters are:-\r\na. target_column - Aggregation column\r\nb. window_size - aggregation window size\r\nc. window unit - aggregation window unit (ex - day, hour)\r\nd. lateral_view_expression - definition of a lateral view for the feature.\r\ne. lateral_view_table_alias - An alias for the lateral view\r\nf. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression.\r\ng. groupBy - groupBy columns. Should be a sparkSql expression.*/parameters:optional map[string,string]}", SchemaFormatType.PDL)); - private String _operatorField = null; - private StringMap _parametersField = null; - private AggregationFunction.ChangeListener __changeListener = new AggregationFunction.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Operator = SCHEMA.getField("operator"); - private final static RecordDataSchema.Field FIELD_Parameters = SCHEMA.getField("parameters"); - - public AggregationFunction() { - super(new DataMap(3, 0.75F), SCHEMA, 2); - addChangeListener(__changeListener); - } - - public AggregationFunction(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static AggregationFunction.Fields fields() { - return _fields; - } - - public static AggregationFunction.ProjectionMask createMask() { - return new AggregationFunction.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for operator - * - * @see AggregationFunction.Fields#operator - */ - public boolean hasOperator() { - if (_operatorField!= null) { - return true; - } - return super._map.containsKey("operator"); - } - - /** - * Remover for operator - * - * @see AggregationFunction.Fields#operator - */ - public void removeOperator() { - super._map.remove("operator"); - } - - /** - * Getter for operator - * - * @see AggregationFunction.Fields#operator - */ - public String getOperator(GetMode mode) { - switch (mode) { - case STRICT: - return getOperator(); - case DEFAULT: - case NULL: - if (_operatorField!= null) { - return _operatorField; - } else { - Object __rawValue = super._map.get("operator"); - _operatorField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _operatorField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for operator - * - * @return - * Required field. Could be null for partial record. - * @see AggregationFunction.Fields#operator - */ - @Nonnull - public String getOperator() { - if (_operatorField!= null) { - return _operatorField; - } else { - Object __rawValue = super._map.get("operator"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("operator"); - } - _operatorField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _operatorField; - } - } - - /** - * Setter for operator - * - * @see AggregationFunction.Fields#operator - */ - public AggregationFunction setOperator(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setOperator(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field operator of com.linkedin.feathr.compute.AggregationFunction"); - } else { - CheckedUtil.putWithoutChecking(super._map, "operator", value); - _operatorField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeOperator(); - } else { - CheckedUtil.putWithoutChecking(super._map, "operator", value); - _operatorField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "operator", value); - _operatorField = value; - } - break; - } - return this; - } - - /** - * Setter for operator - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see AggregationFunction.Fields#operator - */ - public AggregationFunction setOperator( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field operator of com.linkedin.feathr.compute.AggregationFunction to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "operator", value); - _operatorField = value; - } - return this; - } - - /** - * Existence checker for parameters - * - * @see AggregationFunction.Fields#parameters - */ - public boolean hasParameters() { - if (_parametersField!= null) { - return true; - } - return super._map.containsKey("parameters"); - } - - /** - * Remover for parameters - * - * @see AggregationFunction.Fields#parameters - */ - public void removeParameters() { - super._map.remove("parameters"); - } - - /** - * Getter for parameters - * - * @see AggregationFunction.Fields#parameters - */ - public StringMap getParameters(GetMode mode) { - return getParameters(); - } - - /** - * Getter for parameters - * - * @return - * Optional field. Always check for null. - * @see AggregationFunction.Fields#parameters - */ - @Nullable - public StringMap getParameters() { - if (_parametersField!= null) { - return _parametersField; - } else { - Object __rawValue = super._map.get("parameters"); - _parametersField = ((__rawValue == null)?null:new StringMap(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _parametersField; - } - } - - /** - * Setter for parameters - * - * @see AggregationFunction.Fields#parameters - */ - public AggregationFunction setParameters(StringMap value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setParameters(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeParameters(); - } else { - CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); - _parametersField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); - _parametersField = value; - } - break; - } - return this; - } - - /** - * Setter for parameters - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see AggregationFunction.Fields#parameters - */ - public AggregationFunction setParameters( - @Nonnull - StringMap value) { - if (value == null) { - throw new NullPointerException("Cannot set field parameters of com.linkedin.feathr.compute.AggregationFunction to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); - _parametersField = value; - } - return this; - } - - @Override - public AggregationFunction clone() - throws CloneNotSupportedException - { - AggregationFunction __clone = ((AggregationFunction) super.clone()); - __clone.__changeListener = new AggregationFunction.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public AggregationFunction copy() - throws CloneNotSupportedException - { - AggregationFunction __copy = ((AggregationFunction) super.copy()); - __copy._parametersField = null; - __copy._operatorField = null; - __copy.__changeListener = new AggregationFunction.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final AggregationFunction __objectRef; - - private ChangeListener(AggregationFunction reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "parameters": - __objectRef._parametersField = null; - break; - case "operator": - __objectRef._operatorField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The aggregation function. - * - */ - public PathSpec operator() { - return new PathSpec(getPathComponents(), "operator"); - } - - /** - * All the aggregation parameters should be bundled into this map. For now, the possible parameters are:- - * a. target_column - Aggregation column - * b. window_size - aggregation window size - * c. window unit - aggregation window unit (ex - day, hour) - * d. lateral_view_expression - definition of a lateral view for the feature. - * e. lateral_view_table_alias - An alias for the lateral view - * f. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression. - * g. groupBy - groupBy columns. Should be a sparkSql expression. - * - */ - public PathSpec parameters() { - return new PathSpec(getPathComponents(), "parameters"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(3); - } - - /** - * The aggregation function. - * - */ - public AggregationFunction.ProjectionMask withOperator() { - getDataMap().put("operator", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * All the aggregation parameters should be bundled into this map. For now, the possible parameters are:- - * a. target_column - Aggregation column - * b. window_size - aggregation window size - * c. window unit - aggregation window unit (ex - day, hour) - * d. lateral_view_expression - definition of a lateral view for the feature. - * e. lateral_view_table_alias - An alias for the lateral view - * f. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression. - * g. groupBy - groupBy columns. Should be a sparkSql expression. - * - */ - public AggregationFunction.ProjectionMask withParameters() { - getDataMap().put("parameters", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationType.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationType.java deleted file mode 100644 index fc3cc0284..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AggregationType.java +++ /dev/null @@ -1,78 +0,0 @@ - -package com.linkedin.feathr.compute; - -import javax.annotation.Generated; -import com.linkedin.data.schema.EnumDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; - - -/** - * - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SlidingWindowFeature.pdl.") -public enum AggregationType { - - - /** - * Sum. - * - */ - SUM, - - /** - * Count. - * - */ - COUNT, - - /** - * Max. - * - */ - MAX, - - /** - * Min. - * - */ - MIN, - - /** - * Average. - * - */ - AVG, - - /** - * Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Max pooling is done by applying a max filter to (usually) non-overlapping subregions of the initial representation. - * - */ - MAX_POOLING, - - /** - * Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Min pooling is done by applying a min filter to (usually) non-overlapping subregions of the initial representation. - * - */ - MIN_POOLING, - - /** - * Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Average pooling is done by applying a average filter to (usually) non-overlapping subregions of the initial representation. - * - */ - AVG_POOLING, - - /** - * Latest - * - */ - LATEST, - $UNKNOWN; - private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute,enum AggregationType{/** Sum. */SUM/** Count. */COUNT/** Max. */MAX/** Min. */MIN/** Average. */AVG/** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Max pooling is done by applying a max filter to (usually) non-overlapping subregions of the initial representation. */MAX_POOLING/** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Min pooling is done by applying a min filter to (usually) non-overlapping subregions of the initial representation. */MIN_POOLING/** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Average pooling is done by applying a average filter to (usually) non-overlapping subregions of the initial representation. */AVG_POOLING/** Latest */LATEST}", SchemaFormatType.PDL)); - - public static EnumDataSchema dataSchema() { - return SCHEMA; - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNode.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNode.java deleted file mode 100644 index fe1ec5a8f..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNode.java +++ /dev/null @@ -1,357 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.DataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.TyperefDataSchema; -import com.linkedin.data.schema.UnionDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.HasTyperefInfo; -import com.linkedin.data.template.TyperefInfo; -import com.linkedin.data.template.UnionTemplate; - -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\AnyNode.pdl.") -public class AnyNode - extends UnionTemplate - implements HasTyperefInfo -{ - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**A node to represent an aggregation step. The aggregation inputs like the groupBy field, agg function are delegated to [[AggregationFunction]].\r\nThis node can represent a feature. As of now, in this step we will be using the SWA library from Spark-algorithms.*/record Aggregation includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node.*/input:/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}/**All the aggregation related parameters and functions are bundled into this.*/function:/**All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node.*/record AggregationFunction{/**The aggregation function.*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**All the aggregation parameters should be bundled into this map. For now, the possible parameters are:-\r\na. target_column - Aggregation column\r\nb. window_size - aggregation window size\r\nc. window unit - aggregation window unit (ex - day, hour)\r\nd. lateral_view_expression - definition of a lateral view for the feature.\r\ne. lateral_view_table_alias - An alias for the lateral view\r\nf. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression.\r\ng. groupBy - groupBy columns. Should be a sparkSql expression.*/parameters:optional map[string,string]}/**If the node is representing a feature, the feature name should be associated with the node.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}}{namespace com.linkedin.feathr.compute/**Representation of the datasource node. There are 3 types of datasource nodes:-\r\nContext - To represent the observation data entities (like the join key or passthrough feature columns)\r\nUpdate - To represent a non-timepartitioned datasource node.\r\nEvent - To represent a time-partitioned datasource node.\r\n\r\nTODO - Maybe, it makes sense more sense to refactor it by make this an abstract object, and deriving the three different nodes from it.*/record DataSource includes AbstractNode{/**Type of node, ie - Context, Update, Event*/sourceType:/**Type of datasource node.*/enum DataSourceType{/**Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log.*/UPDATE/**Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K\u00e2\u20ac\u2122d)\r\nover a limited window of time.*/EVENT/**Reprent the observation data entities (like the join key or passthrough feature columns)*/CONTEXT}/**for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI.*/externalSourceRef:string/**Raw key expression as entered by the user. This hocon parsing happens at the execution engine side.*/keyExpression:string/**mvel or spark or user-defined class*/keyExpressionType:/**Different key formats supported.\r\nTodo - We probably do not want to generalize this as a kind of key-operator in the core compute model,\r\nwith instances such as for MVEL or SQL being available (e.g. via an OperatorId reference).*/enum KeyExpressionType{/**Java-based MVEL*/MVEL/**Spark-SQL*/SQL/**Custom java/scala UDF*/UDF}/**File partition format.*/filePartitionFormat:optional string/**Timestamp column info, to be available only for an event datasource node.*/timestampColumnInfo:optional/**Representation of a timestamp column field*/record TimestampCol{/**Timestamp column expression.*/expression:string/**Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis*/format:string}}}{namespace com.linkedin.feathr.compute/**A node to represent a feature which is to be computed by using an already computed feature as the key.\r\nhttps://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?spaceKey=ENGS&title=feathr+Offline+User+Guide#FrameOfflineUserGuide-sequentialjoin*/record Lookup includes AbstractNode{/**An array of references to a node and keys.\r\n\r\nFor now, we do not support lookup of just a key reference, but we have added that as a placeholder.\r\n\r\nA node reference consists of node id and a key reference.\r\nIn sequential join the lookup key would be a combination of the\r\nfeature node representing the base feature (lookup node) and the key associated with it. For example,:-\r\nseqJoinFeature: {\r\n base: {key: x, feature: baseFeature}\r\n expansion: {key: y, feature: expansionFeature}\r\n aggregation: UNION\r\n}\r\nHere, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would\r\npoint to the index of \"x\" in the key array of baseFeature.*/lookupKey:array[union[NodeReference,KeyReference]]/**The node id of the node containing the expansion feature.*/lookupNode:NodeId/**Aggregation type as listed in\r\nhttps://jarvis.corp.linkedin.com/codesearch/result/\r\n?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7\r\n*/aggregation:string/**feature name of the feature which would be computed.\r\nwe need feature name here for 2 main reasons.\r\n1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and\r\nwe want to leverage that.\r\n2. For default values. Similar to above, there are existing APIs which create default value map from feature name ->\r\ndefault value.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}}{namespace com.linkedin.feathr.compute/**Representation of a transformation node.*/record Transformation includes AbstractNode{/**An array of node references which should be considered as input to apply the transformation function.*/inputs:array[NodeReference]/**The transformation function.*/function:/**The transformation function*/record TransformationFunction{/**Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class.\r\n*/operator:OperatorId/**The various attributes required to represent the transformation function are captured in a map format.\r\nFor example, mvel expression or java udf class name*/parameters:optional map[string,string]}/**Feature name here is used so we retain feature name, type, and default values even after graph is resolved.\r\nFeature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}}{namespace com.linkedin.feathr.compute/**A temporary node which would exist only while parsing the graph. For example, when parsing an object if there is a reference to a feature\r\nname, we will create an external node. This would get resolved later in the computation.*/record External includes AbstractNode{/**Name of the external object it should refer to.*/name:string}}]", SchemaFormatType.PDL)); - private com.linkedin.feathr.compute.Aggregation _aggregationMember = null; - private com.linkedin.feathr.compute.DataSource _dataSourceMember = null; - private com.linkedin.feathr.compute.Lookup _lookupMember = null; - private com.linkedin.feathr.compute.Transformation _transformationMember = null; - private com.linkedin.feathr.compute.External _externalMember = null; - private AnyNode.ChangeListener __changeListener = new AnyNode.ChangeListener(this); - private final static DataSchema MEMBER_Aggregation = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.Aggregation"); - private final static DataSchema MEMBER_DataSource = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.DataSource"); - private final static DataSchema MEMBER_Lookup = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.Lookup"); - private final static DataSchema MEMBER_Transformation = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.Transformation"); - private final static DataSchema MEMBER_External = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.External"); - private final static TyperefInfo TYPEREFINFO = new AnyNode.UnionTyperefInfo(); - - public AnyNode() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public AnyNode(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static AnyNode create(com.linkedin.feathr.compute.Aggregation value) { - AnyNode newUnion = new AnyNode(); - newUnion.setAggregation(value); - return newUnion; - } - - public boolean isAggregation() { - return memberIs("com.linkedin.feathr.compute.Aggregation"); - } - - public com.linkedin.feathr.compute.Aggregation getAggregation() { - checkNotNull(); - if (_aggregationMember!= null) { - return _aggregationMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.Aggregation"); - _aggregationMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.Aggregation(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _aggregationMember; - } - - public void setAggregation(com.linkedin.feathr.compute.Aggregation value) { - checkNotNull(); - super._map.clear(); - _aggregationMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.Aggregation", value.data()); - } - - public static AnyNode create(com.linkedin.feathr.compute.DataSource value) { - AnyNode newUnion = new AnyNode(); - newUnion.setDataSource(value); - return newUnion; - } - - public boolean isDataSource() { - return memberIs("com.linkedin.feathr.compute.DataSource"); - } - - public com.linkedin.feathr.compute.DataSource getDataSource() { - checkNotNull(); - if (_dataSourceMember!= null) { - return _dataSourceMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.DataSource"); - _dataSourceMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.DataSource(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _dataSourceMember; - } - - public void setDataSource(com.linkedin.feathr.compute.DataSource value) { - checkNotNull(); - super._map.clear(); - _dataSourceMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.DataSource", value.data()); - } - - public static AnyNode create(com.linkedin.feathr.compute.Lookup value) { - AnyNode newUnion = new AnyNode(); - newUnion.setLookup(value); - return newUnion; - } - - public boolean isLookup() { - return memberIs("com.linkedin.feathr.compute.Lookup"); - } - - public com.linkedin.feathr.compute.Lookup getLookup() { - checkNotNull(); - if (_lookupMember!= null) { - return _lookupMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.Lookup"); - _lookupMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.Lookup(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _lookupMember; - } - - public void setLookup(com.linkedin.feathr.compute.Lookup value) { - checkNotNull(); - super._map.clear(); - _lookupMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.Lookup", value.data()); - } - - public static AnyNode create(com.linkedin.feathr.compute.Transformation value) { - AnyNode newUnion = new AnyNode(); - newUnion.setTransformation(value); - return newUnion; - } - - public boolean isTransformation() { - return memberIs("com.linkedin.feathr.compute.Transformation"); - } - - public com.linkedin.feathr.compute.Transformation getTransformation() { - checkNotNull(); - if (_transformationMember!= null) { - return _transformationMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.Transformation"); - _transformationMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.Transformation(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _transformationMember; - } - - public void setTransformation(com.linkedin.feathr.compute.Transformation value) { - checkNotNull(); - super._map.clear(); - _transformationMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.Transformation", value.data()); - } - - public static AnyNode create(com.linkedin.feathr.compute.External value) { - AnyNode newUnion = new AnyNode(); - newUnion.setExternal(value); - return newUnion; - } - - public boolean isExternal() { - return memberIs("com.linkedin.feathr.compute.External"); - } - - public com.linkedin.feathr.compute.External getExternal() { - checkNotNull(); - if (_externalMember!= null) { - return _externalMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.External"); - _externalMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.External(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _externalMember; - } - - public void setExternal(com.linkedin.feathr.compute.External value) { - checkNotNull(); - super._map.clear(); - _externalMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.External", value.data()); - } - - public static AnyNode.ProjectionMask createMask() { - return new AnyNode.ProjectionMask(); - } - - @Override - public AnyNode clone() - throws CloneNotSupportedException - { - AnyNode __clone = ((AnyNode) super.clone()); - __clone.__changeListener = new AnyNode.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public AnyNode copy() - throws CloneNotSupportedException - { - AnyNode __copy = ((AnyNode) super.copy()); - __copy._lookupMember = null; - __copy._transformationMember = null; - __copy._aggregationMember = null; - __copy._dataSourceMember = null; - __copy._externalMember = null; - __copy.__changeListener = new AnyNode.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - public TyperefInfo typerefInfo() { - return TYPEREFINFO; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final AnyNode __objectRef; - - private ChangeListener(AnyNode reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "com.linkedin.feathr.compute.Lookup": - __objectRef._lookupMember = null; - break; - case "com.linkedin.feathr.compute.Transformation": - __objectRef._transformationMember = null; - break; - case "com.linkedin.feathr.compute.Aggregation": - __objectRef._aggregationMember = null; - break; - case "com.linkedin.feathr.compute.DataSource": - __objectRef._dataSourceMember = null; - break; - case "com.linkedin.feathr.compute.External": - __objectRef._externalMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.compute.Aggregation.Fields Aggregation() { - return new com.linkedin.feathr.compute.Aggregation.Fields(getPathComponents(), "com.linkedin.feathr.compute.Aggregation"); - } - - public com.linkedin.feathr.compute.DataSource.Fields DataSource() { - return new com.linkedin.feathr.compute.DataSource.Fields(getPathComponents(), "com.linkedin.feathr.compute.DataSource"); - } - - public com.linkedin.feathr.compute.Lookup.Fields Lookup() { - return new com.linkedin.feathr.compute.Lookup.Fields(getPathComponents(), "com.linkedin.feathr.compute.Lookup"); - } - - public com.linkedin.feathr.compute.Transformation.Fields Transformation() { - return new com.linkedin.feathr.compute.Transformation.Fields(getPathComponents(), "com.linkedin.feathr.compute.Transformation"); - } - - public com.linkedin.feathr.compute.External.Fields External() { - return new com.linkedin.feathr.compute.External.Fields(getPathComponents(), "com.linkedin.feathr.compute.External"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.Aggregation.ProjectionMask _AggregationMask; - private com.linkedin.feathr.compute.DataSource.ProjectionMask _DataSourceMask; - private com.linkedin.feathr.compute.Lookup.ProjectionMask _LookupMask; - private com.linkedin.feathr.compute.Transformation.ProjectionMask _TransformationMask; - private com.linkedin.feathr.compute.External.ProjectionMask _ExternalMask; - - ProjectionMask() { - super(7); - } - - public AnyNode.ProjectionMask withAggregation(Function nestedMask) { - _AggregationMask = nestedMask.apply(((_AggregationMask == null)?com.linkedin.feathr.compute.Aggregation.createMask():_AggregationMask)); - getDataMap().put("com.linkedin.feathr.compute.Aggregation", _AggregationMask.getDataMap()); - return this; - } - - public AnyNode.ProjectionMask withDataSource(Function nestedMask) { - _DataSourceMask = nestedMask.apply(((_DataSourceMask == null)?com.linkedin.feathr.compute.DataSource.createMask():_DataSourceMask)); - getDataMap().put("com.linkedin.feathr.compute.DataSource", _DataSourceMask.getDataMap()); - return this; - } - - public AnyNode.ProjectionMask withLookup(Function nestedMask) { - _LookupMask = nestedMask.apply(((_LookupMask == null)?com.linkedin.feathr.compute.Lookup.createMask():_LookupMask)); - getDataMap().put("com.linkedin.feathr.compute.Lookup", _LookupMask.getDataMap()); - return this; - } - - public AnyNode.ProjectionMask withTransformation(Function nestedMask) { - _TransformationMask = nestedMask.apply(((_TransformationMask == null)?com.linkedin.feathr.compute.Transformation.createMask():_TransformationMask)); - getDataMap().put("com.linkedin.feathr.compute.Transformation", _TransformationMask.getDataMap()); - return this; - } - - public AnyNode.ProjectionMask withExternal(Function nestedMask) { - _ExternalMask = nestedMask.apply(((_ExternalMask == null)?com.linkedin.feathr.compute.External.createMask():_ExternalMask)); - getDataMap().put("com.linkedin.feathr.compute.External", _ExternalMask.getDataMap()); - return this; - } - - } - - - /** - * A typeref for all the different types of nodes. - * - */ - private final static class UnionTyperefInfo - extends TyperefInfo - { - - private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**A typeref for all the different types of nodes.*/typeref AnyNode=union[/**A node to represent an aggregation step. The aggregation inputs like the groupBy field, agg function are delegated to [[AggregationFunction]].\r\nThis node can represent a feature. As of now, in this step we will be using the SWA library from Spark-algorithms.*/record Aggregation includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node.*/input:/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}/**All the aggregation related parameters and functions are bundled into this.*/function:/**All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node.*/record AggregationFunction{/**The aggregation function.*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**All the aggregation parameters should be bundled into this map. For now, the possible parameters are:-\r\na. target_column - Aggregation column\r\nb. window_size - aggregation window size\r\nc. window unit - aggregation window unit (ex - day, hour)\r\nd. lateral_view_expression - definition of a lateral view for the feature.\r\ne. lateral_view_table_alias - An alias for the lateral view\r\nf. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression.\r\ng. groupBy - groupBy columns. Should be a sparkSql expression.*/parameters:optional map[string,string]}/**If the node is representing a feature, the feature name should be associated with the node.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}/**Representation of the datasource node. There are 3 types of datasource nodes:-\r\nContext - To represent the observation data entities (like the join key or passthrough feature columns)\r\nUpdate - To represent a non-timepartitioned datasource node.\r\nEvent - To represent a time-partitioned datasource node.\r\n\r\nTODO - Maybe, it makes sense more sense to refactor it by make this an abstract object, and deriving the three different nodes from it.*/record DataSource includes AbstractNode{/**Type of node, ie - Context, Update, Event*/sourceType:/**Type of datasource node.*/enum DataSourceType{/**Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log.*/UPDATE/**Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K\u00e2\u20ac\u2122d)\r\nover a limited window of time.*/EVENT/**Reprent the observation data entities (like the join key or passthrough feature columns)*/CONTEXT}/**for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI.*/externalSourceRef:string/**Raw key expression as entered by the user. This hocon parsing happens at the execution engine side.*/keyExpression:string/**mvel or spark or user-defined class*/keyExpressionType:/**Different key formats supported.\r\nTodo - We probably do not want to generalize this as a kind of key-operator in the core compute model,\r\nwith instances such as for MVEL or SQL being available (e.g. via an OperatorId reference).*/enum KeyExpressionType{/**Java-based MVEL*/MVEL/**Spark-SQL*/SQL/**Custom java/scala UDF*/UDF}/**File partition format.*/filePartitionFormat:optional string/**Timestamp column info, to be available only for an event datasource node.*/timestampColumnInfo:optional/**Representation of a timestamp column field*/record TimestampCol{/**Timestamp column expression.*/expression:string/**Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis*/format:string}}/**A node to represent a feature which is to be computed by using an already computed feature as the key.\r\nhttps://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?spaceKey=ENGS&title=feathr+Offline+User+Guide#FrameOfflineUserGuide-sequentialjoin*/record Lookup includes AbstractNode{/**An array of references to a node and keys.\r\n\r\nFor now, we do not support lookup of just a key reference, but we have added that as a placeholder.\r\n\r\nA node reference consists of node id and a key reference.\r\nIn sequential join the lookup key would be a combination of the\r\nfeature node representing the base feature (lookup node) and the key associated with it. For example,:-\r\nseqJoinFeature: {\r\n base: {key: x, feature: baseFeature}\r\n expansion: {key: y, feature: expansionFeature}\r\n aggregation: UNION\r\n}\r\nHere, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would\r\npoint to the index of \"x\" in the key array of baseFeature.*/lookupKey:array[union[NodeReference,KeyReference]]/**The node id of the node containing the expansion feature.*/lookupNode:NodeId/**Aggregation type as listed in\r\nhttps://jarvis.corp.linkedin.com/codesearch/result/\r\n?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7\r\n*/aggregation:string/**feature name of the feature which would be computed.\r\nwe need feature name here for 2 main reasons.\r\n1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and\r\nwe want to leverage that.\r\n2. For default values. Similar to above, there are existing APIs which create default value map from feature name ->\r\ndefault value.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}/**Representation of a transformation node.*/record Transformation includes AbstractNode{/**An array of node references which should be considered as input to apply the transformation function.*/inputs:array[NodeReference]/**The transformation function.*/function:/**The transformation function*/record TransformationFunction{/**Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class.\r\n*/operator:OperatorId/**The various attributes required to represent the transformation function are captured in a map format.\r\nFor example, mvel expression or java udf class name*/parameters:optional map[string,string]}/**Feature name here is used so we retain feature name, type, and default values even after graph is resolved.\r\nFeature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}/**A temporary node which would exist only while parsing the graph. For example, when parsing an object if there is a reference to a feature\r\nname, we will create an external node. This would get resolved later in the computation.*/record External includes AbstractNode{/**Name of the external object it should refer to.*/name:string}]", SchemaFormatType.PDL)); - - public UnionTyperefInfo() { - super(SCHEMA); - } - - public static TyperefDataSchema dataSchema() { - return SCHEMA; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNodeArray.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNodeArray.java deleted file mode 100644 index 92b608259..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/AnyNodeArray.java +++ /dev/null @@ -1,117 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import com.linkedin.data.DataList; -import com.linkedin.data.schema.ArrayDataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.TemplateOutputCastException; -import com.linkedin.data.template.WrappingArrayTemplate; - -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\ComputeGraph.pdl.") -public class AnyNodeArray - extends WrappingArrayTemplate -{ - - private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[union[{namespace com.linkedin.feathr.compute/**A node to represent an aggregation step. The aggregation inputs like the groupBy field, agg function are delegated to [[AggregationFunction]].\r\nThis node can represent a feature. As of now, in this step we will be using the SWA library from Spark-algorithms.*/record Aggregation includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node.*/input:/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}/**All the aggregation related parameters and functions are bundled into this.*/function:/**All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node.*/record AggregationFunction{/**The aggregation function.*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**All the aggregation parameters should be bundled into this map. For now, the possible parameters are:-\r\na. target_column - Aggregation column\r\nb. window_size - aggregation window size\r\nc. window unit - aggregation window unit (ex - day, hour)\r\nd. lateral_view_expression - definition of a lateral view for the feature.\r\ne. lateral_view_table_alias - An alias for the lateral view\r\nf. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression.\r\ng. groupBy - groupBy columns. Should be a sparkSql expression.*/parameters:optional map[string,string]}/**If the node is representing a feature, the feature name should be associated with the node.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}}{namespace com.linkedin.feathr.compute/**Representation of the datasource node. There are 3 types of datasource nodes:-\r\nContext - To represent the observation data entities (like the join key or passthrough feature columns)\r\nUpdate - To represent a non-timepartitioned datasource node.\r\nEvent - To represent a time-partitioned datasource node.\r\n\r\nTODO - Maybe, it makes sense more sense to refactor it by make this an abstract object, and deriving the three different nodes from it.*/record DataSource includes AbstractNode{/**Type of node, ie - Context, Update, Event*/sourceType:/**Type of datasource node.*/enum DataSourceType{/**Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log.*/UPDATE/**Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K\u00e2\u20ac\u2122d)\r\nover a limited window of time.*/EVENT/**Reprent the observation data entities (like the join key or passthrough feature columns)*/CONTEXT}/**for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI.*/externalSourceRef:string/**Raw key expression as entered by the user. This hocon parsing happens at the execution engine side.*/keyExpression:string/**mvel or spark or user-defined class*/keyExpressionType:/**Different key formats supported.\r\nTodo - We probably do not want to generalize this as a kind of key-operator in the core compute model,\r\nwith instances such as for MVEL or SQL being available (e.g. via an OperatorId reference).*/enum KeyExpressionType{/**Java-based MVEL*/MVEL/**Spark-SQL*/SQL/**Custom java/scala UDF*/UDF}/**File partition format.*/filePartitionFormat:optional string/**Timestamp column info, to be available only for an event datasource node.*/timestampColumnInfo:optional/**Representation of a timestamp column field*/record TimestampCol{/**Timestamp column expression.*/expression:string/**Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis*/format:string}}}{namespace com.linkedin.feathr.compute/**A node to represent a feature which is to be computed by using an already computed feature as the key.\r\nhttps://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?spaceKey=ENGS&title=feathr+Offline+User+Guide#FrameOfflineUserGuide-sequentialjoin*/record Lookup includes AbstractNode{/**An array of references to a node and keys.\r\n\r\nFor now, we do not support lookup of just a key reference, but we have added that as a placeholder.\r\n\r\nA node reference consists of node id and a key reference.\r\nIn sequential join the lookup key would be a combination of the\r\nfeature node representing the base feature (lookup node) and the key associated with it. For example,:-\r\nseqJoinFeature: {\r\n base: {key: x, feature: baseFeature}\r\n expansion: {key: y, feature: expansionFeature}\r\n aggregation: UNION\r\n}\r\nHere, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would\r\npoint to the index of \"x\" in the key array of baseFeature.*/lookupKey:array[union[NodeReference,KeyReference]]/**The node id of the node containing the expansion feature.*/lookupNode:NodeId/**Aggregation type as listed in\r\nhttps://jarvis.corp.linkedin.com/codesearch/result/\r\n?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7\r\n*/aggregation:string/**feature name of the feature which would be computed.\r\nwe need feature name here for 2 main reasons.\r\n1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and\r\nwe want to leverage that.\r\n2. For default values. Similar to above, there are existing APIs which create default value map from feature name ->\r\ndefault value.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}}{namespace com.linkedin.feathr.compute/**Representation of a transformation node.*/record Transformation includes AbstractNode{/**An array of node references which should be considered as input to apply the transformation function.*/inputs:array[NodeReference]/**The transformation function.*/function:/**The transformation function*/record TransformationFunction{/**Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class.\r\n*/operator:OperatorId/**The various attributes required to represent the transformation function are captured in a map format.\r\nFor example, mvel expression or java udf class name*/parameters:optional map[string,string]}/**Feature name here is used so we retain feature name, type, and default values even after graph is resolved.\r\nFeature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}}{namespace com.linkedin.feathr.compute/**A temporary node which would exist only while parsing the graph. For example, when parsing an object if there is a reference to a feature\r\nname, we will create an external node. This would get resolved later in the computation.*/record External includes AbstractNode{/**Name of the external object it should refer to.*/name:string}}]]", SchemaFormatType.PDL)); - - public AnyNodeArray() { - this(new DataList()); - } - - public AnyNodeArray(int initialCapacity) { - this(new DataList(initialCapacity)); - } - - public AnyNodeArray(Collection c) { - this(new DataList(c.size())); - addAll(c); - } - - public AnyNodeArray(DataList data) { - super(data, SCHEMA, AnyNode.class); - } - - public AnyNodeArray(AnyNode first, AnyNode... rest) { - this(new DataList((rest.length + 1))); - add(first); - addAll(Arrays.asList(rest)); - } - - public static ArrayDataSchema dataSchema() { - return SCHEMA; - } - - public static AnyNodeArray.ProjectionMask createMask() { - return new AnyNodeArray.ProjectionMask(); - } - - @Override - public AnyNodeArray clone() - throws CloneNotSupportedException - { - AnyNodeArray __clone = ((AnyNodeArray) super.clone()); - return __clone; - } - - @Override - public AnyNodeArray copy() - throws CloneNotSupportedException - { - AnyNodeArray __copy = ((AnyNodeArray) super.copy()); - return __copy; - } - - @Override - protected AnyNode coerceOutput(Object object) - throws TemplateOutputCastException - { - assert(object != null); - return ((object == null)?null:new AnyNode(object)); - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.compute.AnyNode.Fields items() { - return new com.linkedin.feathr.compute.AnyNode.Fields(getPathComponents(), PathSpec.WILDCARD); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.AnyNode.ProjectionMask _itemsMask; - - ProjectionMask() { - super(4); - } - - public AnyNodeArray.ProjectionMask withItems(Function nestedMask) { - _itemsMask = nestedMask.apply(((_itemsMask == null)?AnyNode.createMask():_itemsMask)); - getDataMap().put("$*", _itemsMask.getDataMap()); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ComputeGraph.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ComputeGraph.java deleted file mode 100644 index 8258206ea..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ComputeGraph.java +++ /dev/null @@ -1,464 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.IntegerMap; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * Feature computation graph. The passed in feature definition graph should get converted to this dependency graph. This graph is a - * direct translation of all the features present, and is not optimized with respect to the join config. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\ComputeGraph.pdl.") -public class ComputeGraph - extends RecordTemplate -{ - - private final static ComputeGraph.Fields _fields = new ComputeGraph.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Feature computation graph. The passed in feature definition graph should get converted to this dependency graph. This graph is a\r\ndirect translation of all the features present, and is not optimized with respect to the join config.*/record ComputeGraph{/**The nodes in the graph (order does not matter)*/nodes:array[/**A typeref for all the different types of nodes.*/typeref AnyNode=union[/**A node to represent an aggregation step. The aggregation inputs like the groupBy field, agg function are delegated to [[AggregationFunction]].\r\nThis node can represent a feature. As of now, in this step we will be using the SWA library from Spark-algorithms.*/record Aggregation includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**The input node on which aggregation is to be performed. As of now, we would only be supporting this node to be a data source node.*/input:/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}/**All the aggregation related parameters and functions are bundled into this.*/function:/**All parameters related to an aggregation operation. This class should be used in conjunction with the [[Aggregation]] node.*/record AggregationFunction{/**The aggregation function.*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**All the aggregation parameters should be bundled into this map. For now, the possible parameters are:-\r\na. target_column - Aggregation column\r\nb. window_size - aggregation window size\r\nc. window unit - aggregation window unit (ex - day, hour)\r\nd. lateral_view_expression - definition of a lateral view for the feature.\r\ne. lateral_view_table_alias - An alias for the lateral view\r\nf. filter - An expression to filter out any data before aggregation. Should be a sparkSql expression.\r\ng. groupBy - groupBy columns. Should be a sparkSql expression.*/parameters:optional map[string,string]}/**If the node is representing a feature, the feature name should be associated with the node.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}/**Representation of the datasource node. There are 3 types of datasource nodes:-\r\nContext - To represent the observation data entities (like the join key or passthrough feature columns)\r\nUpdate - To represent a non-timepartitioned datasource node.\r\nEvent - To represent a time-partitioned datasource node.\r\n\r\nTODO - Maybe, it makes sense more sense to refactor it by make this an abstract object, and deriving the three different nodes from it.*/record DataSource includes AbstractNode{/**Type of node, ie - Context, Update, Event*/sourceType:/**Type of datasource node.*/enum DataSourceType{/**Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log.*/UPDATE/**Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K\u00e2\u20ac\u2122d)\r\nover a limited window of time.*/EVENT/**Reprent the observation data entities (like the join key or passthrough feature columns)*/CONTEXT}/**for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI.*/externalSourceRef:string/**Raw key expression as entered by the user. This hocon parsing happens at the execution engine side.*/keyExpression:string/**mvel or spark or user-defined class*/keyExpressionType:/**Different key formats supported.\r\nTodo - We probably do not want to generalize this as a kind of key-operator in the core compute model,\r\nwith instances such as for MVEL or SQL being available (e.g. via an OperatorId reference).*/enum KeyExpressionType{/**Java-based MVEL*/MVEL/**Spark-SQL*/SQL/**Custom java/scala UDF*/UDF}/**File partition format.*/filePartitionFormat:optional string/**Timestamp column info, to be available only for an event datasource node.*/timestampColumnInfo:optional/**Representation of a timestamp column field*/record TimestampCol{/**Timestamp column expression.*/expression:string/**Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis*/format:string}}/**A node to represent a feature which is to be computed by using an already computed feature as the key.\r\nhttps://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?spaceKey=ENGS&title=feathr+Offline+User+Guide#FrameOfflineUserGuide-sequentialjoin*/record Lookup includes AbstractNode{/**An array of references to a node and keys.\r\n\r\nFor now, we do not support lookup of just a key reference, but we have added that as a placeholder.\r\n\r\nA node reference consists of node id and a key reference.\r\nIn sequential join the lookup key would be a combination of the\r\nfeature node representing the base feature (lookup node) and the key associated with it. For example,:-\r\nseqJoinFeature: {\r\n base: {key: x, feature: baseFeature}\r\n expansion: {key: y, feature: expansionFeature}\r\n aggregation: UNION\r\n}\r\nHere, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would\r\npoint to the index of \"x\" in the key array of baseFeature.*/lookupKey:array[union[NodeReference,KeyReference]]/**The node id of the node containing the expansion feature.*/lookupNode:NodeId/**Aggregation type as listed in\r\nhttps://jarvis.corp.linkedin.com/codesearch/result/\r\n?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7\r\n*/aggregation:string/**feature name of the feature which would be computed.\r\nwe need feature name here for 2 main reasons.\r\n1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and\r\nwe want to leverage that.\r\n2. For default values. Similar to above, there are existing APIs which create default value map from feature name ->\r\ndefault value.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}/**Representation of a transformation node.*/record Transformation includes AbstractNode{/**An array of node references which should be considered as input to apply the transformation function.*/inputs:array[NodeReference]/**The transformation function.*/function:/**The transformation function*/record TransformationFunction{/**Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class.\r\n*/operator:OperatorId/**The various attributes required to represent the transformation function are captured in a map format.\r\nFor example, mvel expression or java udf class name*/parameters:optional map[string,string]}/**Feature name here is used so we retain feature name, type, and default values even after graph is resolved.\r\nFeature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias.*/featureName:string/**feature version of the feature*/featureVersion:FeatureVersion}/**A temporary node which would exist only while parsing the graph. For example, when parsing an object if there is a reference to a feature\r\nname, we will create an external node. This would get resolved later in the computation.*/record External includes AbstractNode{/**Name of the external object it should refer to.*/name:string}]]/**Map from feature name to node ID, for those nodes in the graph that represent named features.*/featureNames:map[string,int]}", SchemaFormatType.PDL)); - private AnyNodeArray _nodesField = null; - private IntegerMap _featureNamesField = null; - private ComputeGraph.ChangeListener __changeListener = new ComputeGraph.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Nodes = SCHEMA.getField("nodes"); - private final static RecordDataSchema.Field FIELD_FeatureNames = SCHEMA.getField("featureNames"); - - public ComputeGraph() { - super(new DataMap(3, 0.75F), SCHEMA, 3); - addChangeListener(__changeListener); - } - - public ComputeGraph(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static ComputeGraph.Fields fields() { - return _fields; - } - - public static ComputeGraph.ProjectionMask createMask() { - return new ComputeGraph.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for nodes - * - * @see ComputeGraph.Fields#nodes - */ - public boolean hasNodes() { - if (_nodesField!= null) { - return true; - } - return super._map.containsKey("nodes"); - } - - /** - * Remover for nodes - * - * @see ComputeGraph.Fields#nodes - */ - public void removeNodes() { - super._map.remove("nodes"); - } - - /** - * Getter for nodes - * - * @see ComputeGraph.Fields#nodes - */ - public AnyNodeArray getNodes(GetMode mode) { - switch (mode) { - case STRICT: - return getNodes(); - case DEFAULT: - case NULL: - if (_nodesField!= null) { - return _nodesField; - } else { - Object __rawValue = super._map.get("nodes"); - _nodesField = ((__rawValue == null)?null:new AnyNodeArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _nodesField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for nodes - * - * @return - * Required field. Could be null for partial record. - * @see ComputeGraph.Fields#nodes - */ - @Nonnull - public AnyNodeArray getNodes() { - if (_nodesField!= null) { - return _nodesField; - } else { - Object __rawValue = super._map.get("nodes"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("nodes"); - } - _nodesField = ((__rawValue == null)?null:new AnyNodeArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _nodesField; - } - } - - /** - * Setter for nodes - * - * @see ComputeGraph.Fields#nodes - */ - public ComputeGraph setNodes(AnyNodeArray value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setNodes(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field nodes of com.linkedin.feathr.compute.ComputeGraph"); - } else { - CheckedUtil.putWithoutChecking(super._map, "nodes", value.data()); - _nodesField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeNodes(); - } else { - CheckedUtil.putWithoutChecking(super._map, "nodes", value.data()); - _nodesField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "nodes", value.data()); - _nodesField = value; - } - break; - } - return this; - } - - /** - * Setter for nodes - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see ComputeGraph.Fields#nodes - */ - public ComputeGraph setNodes( - @Nonnull - AnyNodeArray value) { - if (value == null) { - throw new NullPointerException("Cannot set field nodes of com.linkedin.feathr.compute.ComputeGraph to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "nodes", value.data()); - _nodesField = value; - } - return this; - } - - /** - * Existence checker for featureNames - * - * @see ComputeGraph.Fields#featureNames - */ - public boolean hasFeatureNames() { - if (_featureNamesField!= null) { - return true; - } - return super._map.containsKey("featureNames"); - } - - /** - * Remover for featureNames - * - * @see ComputeGraph.Fields#featureNames - */ - public void removeFeatureNames() { - super._map.remove("featureNames"); - } - - /** - * Getter for featureNames - * - * @see ComputeGraph.Fields#featureNames - */ - public IntegerMap getFeatureNames(GetMode mode) { - switch (mode) { - case STRICT: - return getFeatureNames(); - case DEFAULT: - case NULL: - if (_featureNamesField!= null) { - return _featureNamesField; - } else { - Object __rawValue = super._map.get("featureNames"); - _featureNamesField = ((__rawValue == null)?null:new IntegerMap(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _featureNamesField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for featureNames - * - * @return - * Required field. Could be null for partial record. - * @see ComputeGraph.Fields#featureNames - */ - @Nonnull - public IntegerMap getFeatureNames() { - if (_featureNamesField!= null) { - return _featureNamesField; - } else { - Object __rawValue = super._map.get("featureNames"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("featureNames"); - } - _featureNamesField = ((__rawValue == null)?null:new IntegerMap(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _featureNamesField; - } - } - - /** - * Setter for featureNames - * - * @see ComputeGraph.Fields#featureNames - */ - public ComputeGraph setFeatureNames(IntegerMap value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFeatureNames(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field featureNames of com.linkedin.feathr.compute.ComputeGraph"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureNames", value.data()); - _featureNamesField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeFeatureNames(); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureNames", value.data()); - _featureNamesField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "featureNames", value.data()); - _featureNamesField = value; - } - break; - } - return this; - } - - /** - * Setter for featureNames - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see ComputeGraph.Fields#featureNames - */ - public ComputeGraph setFeatureNames( - @Nonnull - IntegerMap value) { - if (value == null) { - throw new NullPointerException("Cannot set field featureNames of com.linkedin.feathr.compute.ComputeGraph to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureNames", value.data()); - _featureNamesField = value; - } - return this; - } - - @Override - public ComputeGraph clone() - throws CloneNotSupportedException - { - ComputeGraph __clone = ((ComputeGraph) super.clone()); - __clone.__changeListener = new ComputeGraph.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public ComputeGraph copy() - throws CloneNotSupportedException - { - ComputeGraph __copy = ((ComputeGraph) super.copy()); - __copy._nodesField = null; - __copy._featureNamesField = null; - __copy.__changeListener = new ComputeGraph.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final ComputeGraph __objectRef; - - private ChangeListener(ComputeGraph reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "nodes": - __objectRef._nodesField = null; - break; - case "featureNames": - __objectRef._featureNamesField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The nodes in the graph (order does not matter) - * - */ - public com.linkedin.feathr.compute.AnyNodeArray.Fields nodes() { - return new com.linkedin.feathr.compute.AnyNodeArray.Fields(getPathComponents(), "nodes"); - } - - /** - * The nodes in the graph (order does not matter) - * - */ - public PathSpec nodes(Integer start, Integer count) { - PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "nodes"); - if (start!= null) { - arrayPathSpec.setAttribute("start", start); - } - if (count!= null) { - arrayPathSpec.setAttribute("count", count); - } - return arrayPathSpec; - } - - /** - * Map from feature name to node ID, for those nodes in the graph that represent named features. - * - */ - public PathSpec featureNames() { - return new PathSpec(getPathComponents(), "featureNames"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.AnyNodeArray.ProjectionMask _nodesMask; - - ProjectionMask() { - super(3); - } - - /** - * The nodes in the graph (order does not matter) - * - */ - public ComputeGraph.ProjectionMask withNodes(Function nestedMask) { - _nodesMask = nestedMask.apply(((_nodesMask == null)?AnyNodeArray.createMask():_nodesMask)); - getDataMap().put("nodes", _nodesMask.getDataMap()); - return this; - } - - /** - * The nodes in the graph (order does not matter) - * - */ - public ComputeGraph.ProjectionMask withNodes() { - _nodesMask = null; - getDataMap().put("nodes", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * The nodes in the graph (order does not matter) - * - */ - public ComputeGraph.ProjectionMask withNodes(Function nestedMask, Integer start, Integer count) { - _nodesMask = nestedMask.apply(((_nodesMask == null)?AnyNodeArray.createMask():_nodesMask)); - getDataMap().put("nodes", _nodesMask.getDataMap()); - if (start!= null) { - getDataMap().getDataMap("nodes").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("nodes").put("$count", count); - } - return this; - } - - /** - * The nodes in the graph (order does not matter) - * - */ - public ComputeGraph.ProjectionMask withNodes(Integer start, Integer count) { - _nodesMask = null; - getDataMap().put("nodes", new DataMap(3)); - if (start!= null) { - getDataMap().getDataMap("nodes").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("nodes").put("$count", count); - } - return this; - } - - /** - * Map from feature name to node ID, for those nodes in the graph that represent named features. - * - */ - public ComputeGraph.ProjectionMask withFeatureNames() { - getDataMap().put("featureNames", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ConcreteKey.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ConcreteKey.java deleted file mode 100644 index f49dd9894..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ConcreteKey.java +++ /dev/null @@ -1,300 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.IntegerArray; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * The key (node) for which the node in question is requested. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\ConcreteKey.pdl.") -public class ConcreteKey - extends RecordTemplate -{ - - private final static ConcreteKey.Fields _fields = new ConcreteKey.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[/**A type ref to int node id*/typeref NodeId=int]}", SchemaFormatType.PDL)); - private IntegerArray _keyField = null; - private ConcreteKey.ChangeListener __changeListener = new ConcreteKey.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Key = SCHEMA.getField("key"); - - public ConcreteKey() { - super(new DataMap(2, 0.75F), SCHEMA, 2); - addChangeListener(__changeListener); - } - - public ConcreteKey(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static ConcreteKey.Fields fields() { - return _fields; - } - - public static ConcreteKey.ProjectionMask createMask() { - return new ConcreteKey.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for key - * - * @see ConcreteKey.Fields#key - */ - public boolean hasKey() { - if (_keyField!= null) { - return true; - } - return super._map.containsKey("key"); - } - - /** - * Remover for key - * - * @see ConcreteKey.Fields#key - */ - public void removeKey() { - super._map.remove("key"); - } - - /** - * Getter for key - * - * @see ConcreteKey.Fields#key - */ - public IntegerArray getKey(GetMode mode) { - switch (mode) { - case STRICT: - return getKey(); - case DEFAULT: - case NULL: - if (_keyField!= null) { - return _keyField; - } else { - Object __rawValue = super._map.get("key"); - _keyField = ((__rawValue == null)?null:new IntegerArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _keyField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for key - * - * @return - * Required field. Could be null for partial record. - * @see ConcreteKey.Fields#key - */ - @Nonnull - public IntegerArray getKey() { - if (_keyField!= null) { - return _keyField; - } else { - Object __rawValue = super._map.get("key"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("key"); - } - _keyField = ((__rawValue == null)?null:new IntegerArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _keyField; - } - } - - /** - * Setter for key - * - * @see ConcreteKey.Fields#key - */ - public ConcreteKey setKey(IntegerArray value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setKey(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field key of com.linkedin.feathr.compute.ConcreteKey"); - } else { - CheckedUtil.putWithoutChecking(super._map, "key", value.data()); - _keyField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeKey(); - } else { - CheckedUtil.putWithoutChecking(super._map, "key", value.data()); - _keyField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "key", value.data()); - _keyField = value; - } - break; - } - return this; - } - - /** - * Setter for key - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see ConcreteKey.Fields#key - */ - public ConcreteKey setKey( - @Nonnull - IntegerArray value) { - if (value == null) { - throw new NullPointerException("Cannot set field key of com.linkedin.feathr.compute.ConcreteKey to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "key", value.data()); - _keyField = value; - } - return this; - } - - @Override - public ConcreteKey clone() - throws CloneNotSupportedException - { - ConcreteKey __clone = ((ConcreteKey) super.clone()); - __clone.__changeListener = new ConcreteKey.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public ConcreteKey copy() - throws CloneNotSupportedException - { - ConcreteKey __copy = ((ConcreteKey) super.copy()); - __copy._keyField = null; - __copy.__changeListener = new ConcreteKey.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final ConcreteKey __objectRef; - - private ChangeListener(ConcreteKey reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "key": - __objectRef._keyField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x. - * The main exception would be for a Lookup feature, in which case it would point to another node where the lookup - * key gets computed. - * - */ - public PathSpec key() { - return new PathSpec(getPathComponents(), "key"); - } - - /** - * Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x. - * The main exception would be for a Lookup feature, in which case it would point to another node where the lookup - * key gets computed. - * - */ - public PathSpec key(Integer start, Integer count) { - PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "key"); - if (start!= null) { - arrayPathSpec.setAttribute("start", start); - } - if (count!= null) { - arrayPathSpec.setAttribute("count", count); - } - return arrayPathSpec; - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(2); - } - - /** - * Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x. - * The main exception would be for a Lookup feature, in which case it would point to another node where the lookup - * key gets computed. - * - */ - public ConcreteKey.ProjectionMask withKey() { - getDataMap().put("key", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x. - * The main exception would be for a Lookup feature, in which case it would point to another node where the lookup - * key gets computed. - * - */ - public ConcreteKey.ProjectionMask withKey(Integer start, Integer count) { - getDataMap().put("key", new DataMap(3)); - if (start!= null) { - getDataMap().getDataMap("key").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("key").put("$count", count); - } - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSource.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSource.java deleted file mode 100644 index 21b44aa26..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSource.java +++ /dev/null @@ -1,1232 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * Representation of the datasource node. There are 3 types of datasource nodes:- - * Context - To represent the observation data entities (like the join key or passthrough feature columns) - * Update - To represent a non-timepartitioned datasource node. - * Event - To represent a time-partitioned datasource node. - * - * TODO - Maybe, it makes sense more sense to refactor it by make this an abstract object, and deriving the three different nodes from it. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\DataSource.pdl.") -public class DataSource - extends RecordTemplate -{ - - private final static DataSource.Fields _fields = new DataSource.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Representation of the datasource node. There are 3 types of datasource nodes:-\r\nContext - To represent the observation data entities (like the join key or passthrough feature columns)\r\nUpdate - To represent a non-timepartitioned datasource node.\r\nEvent - To represent a time-partitioned datasource node.\r\n\r\nTODO - Maybe, it makes sense more sense to refactor it by make this an abstract object, and deriving the three different nodes from it.*/record DataSource includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**Type of node, ie - Context, Update, Event*/sourceType:/**Type of datasource node.*/enum DataSourceType{/**Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log.*/UPDATE/**Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K\u00e2\u20ac\u2122d)\r\nover a limited window of time.*/EVENT/**Reprent the observation data entities (like the join key or passthrough feature columns)*/CONTEXT}/**for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI.*/externalSourceRef:string/**Raw key expression as entered by the user. This hocon parsing happens at the execution engine side.*/keyExpression:string/**mvel or spark or user-defined class*/keyExpressionType:/**Different key formats supported.\r\nTodo - We probably do not want to generalize this as a kind of key-operator in the core compute model,\r\nwith instances such as for MVEL or SQL being available (e.g. via an OperatorId reference).*/enum KeyExpressionType{/**Java-based MVEL*/MVEL/**Spark-SQL*/SQL/**Custom java/scala UDF*/UDF}/**File partition format.*/filePartitionFormat:optional string/**Timestamp column info, to be available only for an event datasource node.*/timestampColumnInfo:optional/**Representation of a timestamp column field*/record TimestampCol{/**Timestamp column expression.*/expression:string/**Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis*/format:string}}", SchemaFormatType.PDL)); - private Integer _idField = null; - private ConcreteKey _concreteKeyField = null; - private DataSourceType _sourceTypeField = null; - private String _externalSourceRefField = null; - private String _keyExpressionField = null; - private KeyExpressionType _keyExpressionTypeField = null; - private String _filePartitionFormatField = null; - private TimestampCol _timestampColumnInfoField = null; - private DataSource.ChangeListener __changeListener = new DataSource.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); - private final static RecordDataSchema.Field FIELD_ConcreteKey = SCHEMA.getField("concreteKey"); - private final static RecordDataSchema.Field FIELD_SourceType = SCHEMA.getField("sourceType"); - private final static RecordDataSchema.Field FIELD_ExternalSourceRef = SCHEMA.getField("externalSourceRef"); - private final static RecordDataSchema.Field FIELD_KeyExpression = SCHEMA.getField("keyExpression"); - private final static RecordDataSchema.Field FIELD_KeyExpressionType = SCHEMA.getField("keyExpressionType"); - private final static RecordDataSchema.Field FIELD_FilePartitionFormat = SCHEMA.getField("filePartitionFormat"); - private final static RecordDataSchema.Field FIELD_TimestampColumnInfo = SCHEMA.getField("timestampColumnInfo"); - - public DataSource() { - super(new DataMap(11, 0.75F), SCHEMA, 3); - addChangeListener(__changeListener); - } - - public DataSource(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static DataSource.Fields fields() { - return _fields; - } - - public static DataSource.ProjectionMask createMask() { - return new DataSource.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for id - * - * @see DataSource.Fields#id - */ - public boolean hasId() { - if (_idField!= null) { - return true; - } - return super._map.containsKey("id"); - } - - /** - * Remover for id - * - * @see DataSource.Fields#id - */ - public void removeId() { - super._map.remove("id"); - } - - /** - * Getter for id - * - * @see DataSource.Fields#id - */ - public Integer getId(GetMode mode) { - switch (mode) { - case STRICT: - return getId(); - case DEFAULT: - case NULL: - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for id - * - * @return - * Required field. Could be null for partial record. - * @see DataSource.Fields#id - */ - @Nonnull - public Integer getId() { - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("id"); - } - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - - /** - * Setter for id - * - * @see DataSource.Fields#id - */ - public DataSource setId(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setId(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.DataSource"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeId(); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - } - return this; - } - - /** - * Setter for id - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see DataSource.Fields#id - */ - public DataSource setId( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.DataSource to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - return this; - } - - /** - * Setter for id - * - * @see DataSource.Fields#id - */ - public DataSource setId(int value) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - return this; - } - - /** - * Existence checker for concreteKey - * - * @see DataSource.Fields#concreteKey - */ - public boolean hasConcreteKey() { - if (_concreteKeyField!= null) { - return true; - } - return super._map.containsKey("concreteKey"); - } - - /** - * Remover for concreteKey - * - * @see DataSource.Fields#concreteKey - */ - public void removeConcreteKey() { - super._map.remove("concreteKey"); - } - - /** - * Getter for concreteKey - * - * @see DataSource.Fields#concreteKey - */ - public ConcreteKey getConcreteKey(GetMode mode) { - return getConcreteKey(); - } - - /** - * Getter for concreteKey - * - * @return - * Optional field. Always check for null. - * @see DataSource.Fields#concreteKey - */ - @Nullable - public ConcreteKey getConcreteKey() { - if (_concreteKeyField!= null) { - return _concreteKeyField; - } else { - Object __rawValue = super._map.get("concreteKey"); - _concreteKeyField = ((__rawValue == null)?null:new ConcreteKey(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _concreteKeyField; - } - } - - /** - * Setter for concreteKey - * - * @see DataSource.Fields#concreteKey - */ - public DataSource setConcreteKey(ConcreteKey value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setConcreteKey(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeConcreteKey(); - } else { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - break; - } - return this; - } - - /** - * Setter for concreteKey - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see DataSource.Fields#concreteKey - */ - public DataSource setConcreteKey( - @Nonnull - ConcreteKey value) { - if (value == null) { - throw new NullPointerException("Cannot set field concreteKey of com.linkedin.feathr.compute.DataSource to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - return this; - } - - /** - * Existence checker for sourceType - * - * @see DataSource.Fields#sourceType - */ - public boolean hasSourceType() { - if (_sourceTypeField!= null) { - return true; - } - return super._map.containsKey("sourceType"); - } - - /** - * Remover for sourceType - * - * @see DataSource.Fields#sourceType - */ - public void removeSourceType() { - super._map.remove("sourceType"); - } - - /** - * Getter for sourceType - * - * @see DataSource.Fields#sourceType - */ - public DataSourceType getSourceType(GetMode mode) { - switch (mode) { - case STRICT: - return getSourceType(); - case DEFAULT: - case NULL: - if (_sourceTypeField!= null) { - return _sourceTypeField; - } else { - Object __rawValue = super._map.get("sourceType"); - _sourceTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, DataSourceType.class, DataSourceType.$UNKNOWN); - return _sourceTypeField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for sourceType - * - * @return - * Required field. Could be null for partial record. - * @see DataSource.Fields#sourceType - */ - @Nonnull - public DataSourceType getSourceType() { - if (_sourceTypeField!= null) { - return _sourceTypeField; - } else { - Object __rawValue = super._map.get("sourceType"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("sourceType"); - } - _sourceTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, DataSourceType.class, DataSourceType.$UNKNOWN); - return _sourceTypeField; - } - } - - /** - * Setter for sourceType - * - * @see DataSource.Fields#sourceType - */ - public DataSource setSourceType(DataSourceType value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setSourceType(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field sourceType of com.linkedin.feathr.compute.DataSource"); - } else { - CheckedUtil.putWithoutChecking(super._map, "sourceType", value.name()); - _sourceTypeField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeSourceType(); - } else { - CheckedUtil.putWithoutChecking(super._map, "sourceType", value.name()); - _sourceTypeField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "sourceType", value.name()); - _sourceTypeField = value; - } - break; - } - return this; - } - - /** - * Setter for sourceType - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see DataSource.Fields#sourceType - */ - public DataSource setSourceType( - @Nonnull - DataSourceType value) { - if (value == null) { - throw new NullPointerException("Cannot set field sourceType of com.linkedin.feathr.compute.DataSource to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "sourceType", value.name()); - _sourceTypeField = value; - } - return this; - } - - /** - * Existence checker for externalSourceRef - * - * @see DataSource.Fields#externalSourceRef - */ - public boolean hasExternalSourceRef() { - if (_externalSourceRefField!= null) { - return true; - } - return super._map.containsKey("externalSourceRef"); - } - - /** - * Remover for externalSourceRef - * - * @see DataSource.Fields#externalSourceRef - */ - public void removeExternalSourceRef() { - super._map.remove("externalSourceRef"); - } - - /** - * Getter for externalSourceRef - * - * @see DataSource.Fields#externalSourceRef - */ - public String getExternalSourceRef(GetMode mode) { - switch (mode) { - case STRICT: - return getExternalSourceRef(); - case DEFAULT: - case NULL: - if (_externalSourceRefField!= null) { - return _externalSourceRefField; - } else { - Object __rawValue = super._map.get("externalSourceRef"); - _externalSourceRefField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _externalSourceRefField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for externalSourceRef - * - * @return - * Required field. Could be null for partial record. - * @see DataSource.Fields#externalSourceRef - */ - @Nonnull - public String getExternalSourceRef() { - if (_externalSourceRefField!= null) { - return _externalSourceRefField; - } else { - Object __rawValue = super._map.get("externalSourceRef"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("externalSourceRef"); - } - _externalSourceRefField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _externalSourceRefField; - } - } - - /** - * Setter for externalSourceRef - * - * @see DataSource.Fields#externalSourceRef - */ - public DataSource setExternalSourceRef(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setExternalSourceRef(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field externalSourceRef of com.linkedin.feathr.compute.DataSource"); - } else { - CheckedUtil.putWithoutChecking(super._map, "externalSourceRef", value); - _externalSourceRefField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeExternalSourceRef(); - } else { - CheckedUtil.putWithoutChecking(super._map, "externalSourceRef", value); - _externalSourceRefField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "externalSourceRef", value); - _externalSourceRefField = value; - } - break; - } - return this; - } - - /** - * Setter for externalSourceRef - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see DataSource.Fields#externalSourceRef - */ - public DataSource setExternalSourceRef( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field externalSourceRef of com.linkedin.feathr.compute.DataSource to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "externalSourceRef", value); - _externalSourceRefField = value; - } - return this; - } - - /** - * Existence checker for keyExpression - * - * @see DataSource.Fields#keyExpression - */ - public boolean hasKeyExpression() { - if (_keyExpressionField!= null) { - return true; - } - return super._map.containsKey("keyExpression"); - } - - /** - * Remover for keyExpression - * - * @see DataSource.Fields#keyExpression - */ - public void removeKeyExpression() { - super._map.remove("keyExpression"); - } - - /** - * Getter for keyExpression - * - * @see DataSource.Fields#keyExpression - */ - public String getKeyExpression(GetMode mode) { - switch (mode) { - case STRICT: - return getKeyExpression(); - case DEFAULT: - case NULL: - if (_keyExpressionField!= null) { - return _keyExpressionField; - } else { - Object __rawValue = super._map.get("keyExpression"); - _keyExpressionField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _keyExpressionField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for keyExpression - * - * @return - * Required field. Could be null for partial record. - * @see DataSource.Fields#keyExpression - */ - @Nonnull - public String getKeyExpression() { - if (_keyExpressionField!= null) { - return _keyExpressionField; - } else { - Object __rawValue = super._map.get("keyExpression"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("keyExpression"); - } - _keyExpressionField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _keyExpressionField; - } - } - - /** - * Setter for keyExpression - * - * @see DataSource.Fields#keyExpression - */ - public DataSource setKeyExpression(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setKeyExpression(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field keyExpression of com.linkedin.feathr.compute.DataSource"); - } else { - CheckedUtil.putWithoutChecking(super._map, "keyExpression", value); - _keyExpressionField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeKeyExpression(); - } else { - CheckedUtil.putWithoutChecking(super._map, "keyExpression", value); - _keyExpressionField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "keyExpression", value); - _keyExpressionField = value; - } - break; - } - return this; - } - - /** - * Setter for keyExpression - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see DataSource.Fields#keyExpression - */ - public DataSource setKeyExpression( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field keyExpression of com.linkedin.feathr.compute.DataSource to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "keyExpression", value); - _keyExpressionField = value; - } - return this; - } - - /** - * Existence checker for keyExpressionType - * - * @see DataSource.Fields#keyExpressionType - */ - public boolean hasKeyExpressionType() { - if (_keyExpressionTypeField!= null) { - return true; - } - return super._map.containsKey("keyExpressionType"); - } - - /** - * Remover for keyExpressionType - * - * @see DataSource.Fields#keyExpressionType - */ - public void removeKeyExpressionType() { - super._map.remove("keyExpressionType"); - } - - /** - * Getter for keyExpressionType - * - * @see DataSource.Fields#keyExpressionType - */ - public KeyExpressionType getKeyExpressionType(GetMode mode) { - switch (mode) { - case STRICT: - return getKeyExpressionType(); - case DEFAULT: - case NULL: - if (_keyExpressionTypeField!= null) { - return _keyExpressionTypeField; - } else { - Object __rawValue = super._map.get("keyExpressionType"); - _keyExpressionTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, KeyExpressionType.class, KeyExpressionType.$UNKNOWN); - return _keyExpressionTypeField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for keyExpressionType - * - * @return - * Required field. Could be null for partial record. - * @see DataSource.Fields#keyExpressionType - */ - @Nonnull - public KeyExpressionType getKeyExpressionType() { - if (_keyExpressionTypeField!= null) { - return _keyExpressionTypeField; - } else { - Object __rawValue = super._map.get("keyExpressionType"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("keyExpressionType"); - } - _keyExpressionTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, KeyExpressionType.class, KeyExpressionType.$UNKNOWN); - return _keyExpressionTypeField; - } - } - - /** - * Setter for keyExpressionType - * - * @see DataSource.Fields#keyExpressionType - */ - public DataSource setKeyExpressionType(KeyExpressionType value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setKeyExpressionType(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field keyExpressionType of com.linkedin.feathr.compute.DataSource"); - } else { - CheckedUtil.putWithoutChecking(super._map, "keyExpressionType", value.name()); - _keyExpressionTypeField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeKeyExpressionType(); - } else { - CheckedUtil.putWithoutChecking(super._map, "keyExpressionType", value.name()); - _keyExpressionTypeField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "keyExpressionType", value.name()); - _keyExpressionTypeField = value; - } - break; - } - return this; - } - - /** - * Setter for keyExpressionType - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see DataSource.Fields#keyExpressionType - */ - public DataSource setKeyExpressionType( - @Nonnull - KeyExpressionType value) { - if (value == null) { - throw new NullPointerException("Cannot set field keyExpressionType of com.linkedin.feathr.compute.DataSource to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "keyExpressionType", value.name()); - _keyExpressionTypeField = value; - } - return this; - } - - /** - * Existence checker for filePartitionFormat - * - * @see DataSource.Fields#filePartitionFormat - */ - public boolean hasFilePartitionFormat() { - if (_filePartitionFormatField!= null) { - return true; - } - return super._map.containsKey("filePartitionFormat"); - } - - /** - * Remover for filePartitionFormat - * - * @see DataSource.Fields#filePartitionFormat - */ - public void removeFilePartitionFormat() { - super._map.remove("filePartitionFormat"); - } - - /** - * Getter for filePartitionFormat - * - * @see DataSource.Fields#filePartitionFormat - */ - public String getFilePartitionFormat(GetMode mode) { - return getFilePartitionFormat(); - } - - /** - * Getter for filePartitionFormat - * - * @return - * Optional field. Always check for null. - * @see DataSource.Fields#filePartitionFormat - */ - @Nullable - public String getFilePartitionFormat() { - if (_filePartitionFormatField!= null) { - return _filePartitionFormatField; - } else { - Object __rawValue = super._map.get("filePartitionFormat"); - _filePartitionFormatField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _filePartitionFormatField; - } - } - - /** - * Setter for filePartitionFormat - * - * @see DataSource.Fields#filePartitionFormat - */ - public DataSource setFilePartitionFormat(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFilePartitionFormat(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeFilePartitionFormat(); - } else { - CheckedUtil.putWithoutChecking(super._map, "filePartitionFormat", value); - _filePartitionFormatField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "filePartitionFormat", value); - _filePartitionFormatField = value; - } - break; - } - return this; - } - - /** - * Setter for filePartitionFormat - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see DataSource.Fields#filePartitionFormat - */ - public DataSource setFilePartitionFormat( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field filePartitionFormat of com.linkedin.feathr.compute.DataSource to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "filePartitionFormat", value); - _filePartitionFormatField = value; - } - return this; - } - - /** - * Existence checker for timestampColumnInfo - * - * @see DataSource.Fields#timestampColumnInfo - */ - public boolean hasTimestampColumnInfo() { - if (_timestampColumnInfoField!= null) { - return true; - } - return super._map.containsKey("timestampColumnInfo"); - } - - /** - * Remover for timestampColumnInfo - * - * @see DataSource.Fields#timestampColumnInfo - */ - public void removeTimestampColumnInfo() { - super._map.remove("timestampColumnInfo"); - } - - /** - * Getter for timestampColumnInfo - * - * @see DataSource.Fields#timestampColumnInfo - */ - public TimestampCol getTimestampColumnInfo(GetMode mode) { - return getTimestampColumnInfo(); - } - - /** - * Getter for timestampColumnInfo - * - * @return - * Optional field. Always check for null. - * @see DataSource.Fields#timestampColumnInfo - */ - @Nullable - public TimestampCol getTimestampColumnInfo() { - if (_timestampColumnInfoField!= null) { - return _timestampColumnInfoField; - } else { - Object __rawValue = super._map.get("timestampColumnInfo"); - _timestampColumnInfoField = ((__rawValue == null)?null:new TimestampCol(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _timestampColumnInfoField; - } - } - - /** - * Setter for timestampColumnInfo - * - * @see DataSource.Fields#timestampColumnInfo - */ - public DataSource setTimestampColumnInfo(TimestampCol value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setTimestampColumnInfo(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeTimestampColumnInfo(); - } else { - CheckedUtil.putWithoutChecking(super._map, "timestampColumnInfo", value.data()); - _timestampColumnInfoField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "timestampColumnInfo", value.data()); - _timestampColumnInfoField = value; - } - break; - } - return this; - } - - /** - * Setter for timestampColumnInfo - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see DataSource.Fields#timestampColumnInfo - */ - public DataSource setTimestampColumnInfo( - @Nonnull - TimestampCol value) { - if (value == null) { - throw new NullPointerException("Cannot set field timestampColumnInfo of com.linkedin.feathr.compute.DataSource to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "timestampColumnInfo", value.data()); - _timestampColumnInfoField = value; - } - return this; - } - - @Override - public DataSource clone() - throws CloneNotSupportedException - { - DataSource __clone = ((DataSource) super.clone()); - __clone.__changeListener = new DataSource.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public DataSource copy() - throws CloneNotSupportedException - { - DataSource __copy = ((DataSource) super.copy()); - __copy._keyExpressionTypeField = null; - __copy._sourceTypeField = null; - __copy._externalSourceRefField = null; - __copy._timestampColumnInfoField = null; - __copy._keyExpressionField = null; - __copy._filePartitionFormatField = null; - __copy._idField = null; - __copy._concreteKeyField = null; - __copy.__changeListener = new DataSource.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final DataSource __objectRef; - - private ChangeListener(DataSource reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "keyExpressionType": - __objectRef._keyExpressionTypeField = null; - break; - case "sourceType": - __objectRef._sourceTypeField = null; - break; - case "externalSourceRef": - __objectRef._externalSourceRefField = null; - break; - case "timestampColumnInfo": - __objectRef._timestampColumnInfoField = null; - break; - case "keyExpression": - __objectRef._keyExpressionField = null; - break; - case "filePartitionFormat": - __objectRef._filePartitionFormatField = null; - break; - case "id": - __objectRef._idField = null; - break; - case "concreteKey": - __objectRef._concreteKeyField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The node would be represented by this id. - * - */ - public PathSpec id() { - return new PathSpec(getPathComponents(), "id"); - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public com.linkedin.feathr.compute.ConcreteKey.Fields concreteKey() { - return new com.linkedin.feathr.compute.ConcreteKey.Fields(getPathComponents(), "concreteKey"); - } - - /** - * Type of node, ie - Context, Update, Event - * - */ - public PathSpec sourceType() { - return new PathSpec(getPathComponents(), "sourceType"); - } - - /** - * for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI. - * - */ - public PathSpec externalSourceRef() { - return new PathSpec(getPathComponents(), "externalSourceRef"); - } - - /** - * Raw key expression as entered by the user. This hocon parsing happens at the execution engine side. - * - */ - public PathSpec keyExpression() { - return new PathSpec(getPathComponents(), "keyExpression"); - } - - /** - * mvel or spark or user-defined class - * - */ - public PathSpec keyExpressionType() { - return new PathSpec(getPathComponents(), "keyExpressionType"); - } - - /** - * File partition format. - * - */ - public PathSpec filePartitionFormat() { - return new PathSpec(getPathComponents(), "filePartitionFormat"); - } - - /** - * Timestamp column info, to be available only for an event datasource node. - * - */ - public com.linkedin.feathr.compute.TimestampCol.Fields timestampColumnInfo() { - return new com.linkedin.feathr.compute.TimestampCol.Fields(getPathComponents(), "timestampColumnInfo"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.ConcreteKey.ProjectionMask _concreteKeyMask; - private com.linkedin.feathr.compute.TimestampCol.ProjectionMask _timestampColumnInfoMask; - - ProjectionMask() { - super(11); - } - - /** - * The node would be represented by this id. - * - */ - public DataSource.ProjectionMask withId() { - getDataMap().put("id", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public DataSource.ProjectionMask withConcreteKey(Function nestedMask) { - _concreteKeyMask = nestedMask.apply(((_concreteKeyMask == null)?ConcreteKey.createMask():_concreteKeyMask)); - getDataMap().put("concreteKey", _concreteKeyMask.getDataMap()); - return this; - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public DataSource.ProjectionMask withConcreteKey() { - _concreteKeyMask = null; - getDataMap().put("concreteKey", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Type of node, ie - Context, Update, Event - * - */ - public DataSource.ProjectionMask withSourceType() { - getDataMap().put("sourceType", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * for CONTEXT type, this is the name of the context column. otherwise, it should be a path or URI. - * - */ - public DataSource.ProjectionMask withExternalSourceRef() { - getDataMap().put("externalSourceRef", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Raw key expression as entered by the user. This hocon parsing happens at the execution engine side. - * - */ - public DataSource.ProjectionMask withKeyExpression() { - getDataMap().put("keyExpression", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * mvel or spark or user-defined class - * - */ - public DataSource.ProjectionMask withKeyExpressionType() { - getDataMap().put("keyExpressionType", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * File partition format. - * - */ - public DataSource.ProjectionMask withFilePartitionFormat() { - getDataMap().put("filePartitionFormat", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Timestamp column info, to be available only for an event datasource node. - * - */ - public DataSource.ProjectionMask withTimestampColumnInfo(Function nestedMask) { - _timestampColumnInfoMask = nestedMask.apply(((_timestampColumnInfoMask == null)?TimestampCol.createMask():_timestampColumnInfoMask)); - getDataMap().put("timestampColumnInfo", _timestampColumnInfoMask.getDataMap()); - return this; - } - - /** - * Timestamp column info, to be available only for an event datasource node. - * - */ - public DataSource.ProjectionMask withTimestampColumnInfo() { - _timestampColumnInfoMask = null; - getDataMap().put("timestampColumnInfo", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSourceType.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSourceType.java deleted file mode 100644 index 01d80c669..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DataSourceType.java +++ /dev/null @@ -1,43 +0,0 @@ - -package com.linkedin.feathr.compute; - -import javax.annotation.Generated; -import com.linkedin.data.schema.EnumDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; - - -/** - * Type of datasource node. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\DataSourceType.pdl.") -public enum DataSourceType { - - - /** - * Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log. - * - */ - UPDATE, - - /** - * Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K’d) - * over a limited window of time. - * - */ - EVENT, - - /** - * Reprent the observation data entities (like the join key or passthrough feature columns) - * - */ - CONTEXT, - $UNKNOWN; - private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Type of datasource node.*/enum DataSourceType{/**Update data sources provide keyed data about entities. A fully specified table data source contains both a snapshot view and an update log.*/UPDATE/**Event data sources are append-only event logs whose records need to be grouped and aggregated (e.g. counted, averaged, top-K\u00e2\u20ac\u2122d)\r\nover a limited window of time.*/EVENT/**Reprent the observation data entities (like the join key or passthrough feature columns)*/CONTEXT}", SchemaFormatType.PDL)); - - public static EnumDataSchema dataSchema() { - return SCHEMA; - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DateTimeInterval.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DateTimeInterval.java deleted file mode 100644 index 016e9d3aa..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DateTimeInterval.java +++ /dev/null @@ -1,377 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.SetMode; - - -/** - * Represent a data time interval - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\DateTimeInterval.pdl.") -public class DateTimeInterval - extends RecordTemplate -{ - - private final static DateTimeInterval.Fields _fields = new DateTimeInterval.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Represent a data time interval*/record DateTimeInterval{/**Represents the inclusive (greater than or equal to) value in which to start the range. This field is optional. An unset field here indicates an open range; for example, if end is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and start is not set, it would indicate times up to, but excluding, 1455309628000. Note that this interpretation was not originally documented. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact.*/start:optional/**Number of milliseconds since midnight, January 1, 1970 UTC. It must be a positive number*/@compliance=\"NONE\"typeref Time=long/**Represents the exclusive (strictly less than) value in which to end the range. This field is optional. An unset field here indicates an open range; for example, if start is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and end is not set, it would mean everything at, or after, 1455309628000. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact.*/end:optional Time}", SchemaFormatType.PDL)); - private Long _startField = null; - private Long _endField = null; - private DateTimeInterval.ChangeListener __changeListener = new DateTimeInterval.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Start = SCHEMA.getField("start"); - private final static RecordDataSchema.Field FIELD_End = SCHEMA.getField("end"); - - public DateTimeInterval() { - super(new DataMap(3, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public DateTimeInterval(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static DateTimeInterval.Fields fields() { - return _fields; - } - - public static DateTimeInterval.ProjectionMask createMask() { - return new DateTimeInterval.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for start - * - * @see DateTimeInterval.Fields#start - */ - public boolean hasStart() { - if (_startField!= null) { - return true; - } - return super._map.containsKey("start"); - } - - /** - * Remover for start - * - * @see DateTimeInterval.Fields#start - */ - public void removeStart() { - super._map.remove("start"); - } - - /** - * Getter for start - * - * @see DateTimeInterval.Fields#start - */ - public Long getStart(GetMode mode) { - return getStart(); - } - - /** - * Getter for start - * - * @return - * Optional field. Always check for null. - * @see DateTimeInterval.Fields#start - */ - @Nullable - public Long getStart() { - if (_startField!= null) { - return _startField; - } else { - Object __rawValue = super._map.get("start"); - _startField = DataTemplateUtil.coerceLongOutput(__rawValue); - return _startField; - } - } - - /** - * Setter for start - * - * @see DateTimeInterval.Fields#start - */ - public DateTimeInterval setStart(Long value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setStart(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeStart(); - } else { - CheckedUtil.putWithoutChecking(super._map, "start", DataTemplateUtil.coerceLongInput(value)); - _startField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "start", DataTemplateUtil.coerceLongInput(value)); - _startField = value; - } - break; - } - return this; - } - - /** - * Setter for start - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see DateTimeInterval.Fields#start - */ - public DateTimeInterval setStart( - @Nonnull - Long value) { - if (value == null) { - throw new NullPointerException("Cannot set field start of com.linkedin.feathr.compute.DateTimeInterval to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "start", DataTemplateUtil.coerceLongInput(value)); - _startField = value; - } - return this; - } - - /** - * Setter for start - * - * @see DateTimeInterval.Fields#start - */ - public DateTimeInterval setStart(long value) { - CheckedUtil.putWithoutChecking(super._map, "start", DataTemplateUtil.coerceLongInput(value)); - _startField = value; - return this; - } - - /** - * Existence checker for end - * - * @see DateTimeInterval.Fields#end - */ - public boolean hasEnd() { - if (_endField!= null) { - return true; - } - return super._map.containsKey("end"); - } - - /** - * Remover for end - * - * @see DateTimeInterval.Fields#end - */ - public void removeEnd() { - super._map.remove("end"); - } - - /** - * Getter for end - * - * @see DateTimeInterval.Fields#end - */ - public Long getEnd(GetMode mode) { - return getEnd(); - } - - /** - * Getter for end - * - * @return - * Optional field. Always check for null. - * @see DateTimeInterval.Fields#end - */ - @Nullable - public Long getEnd() { - if (_endField!= null) { - return _endField; - } else { - Object __rawValue = super._map.get("end"); - _endField = DataTemplateUtil.coerceLongOutput(__rawValue); - return _endField; - } - } - - /** - * Setter for end - * - * @see DateTimeInterval.Fields#end - */ - public DateTimeInterval setEnd(Long value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setEnd(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeEnd(); - } else { - CheckedUtil.putWithoutChecking(super._map, "end", DataTemplateUtil.coerceLongInput(value)); - _endField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "end", DataTemplateUtil.coerceLongInput(value)); - _endField = value; - } - break; - } - return this; - } - - /** - * Setter for end - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see DateTimeInterval.Fields#end - */ - public DateTimeInterval setEnd( - @Nonnull - Long value) { - if (value == null) { - throw new NullPointerException("Cannot set field end of com.linkedin.feathr.compute.DateTimeInterval to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "end", DataTemplateUtil.coerceLongInput(value)); - _endField = value; - } - return this; - } - - /** - * Setter for end - * - * @see DateTimeInterval.Fields#end - */ - public DateTimeInterval setEnd(long value) { - CheckedUtil.putWithoutChecking(super._map, "end", DataTemplateUtil.coerceLongInput(value)); - _endField = value; - return this; - } - - @Override - public DateTimeInterval clone() - throws CloneNotSupportedException - { - DateTimeInterval __clone = ((DateTimeInterval) super.clone()); - __clone.__changeListener = new DateTimeInterval.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public DateTimeInterval copy() - throws CloneNotSupportedException - { - DateTimeInterval __copy = ((DateTimeInterval) super.copy()); - __copy._startField = null; - __copy._endField = null; - __copy.__changeListener = new DateTimeInterval.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final DateTimeInterval __objectRef; - - private ChangeListener(DateTimeInterval reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "start": - __objectRef._startField = null; - break; - case "end": - __objectRef._endField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Represents the inclusive (greater than or equal to) value in which to start the range. This field is optional. An unset field here indicates an open range; for example, if end is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and start is not set, it would indicate times up to, but excluding, 1455309628000. Note that this interpretation was not originally documented. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact. - * - */ - public PathSpec start() { - return new PathSpec(getPathComponents(), "start"); - } - - /** - * Represents the exclusive (strictly less than) value in which to end the range. This field is optional. An unset field here indicates an open range; for example, if start is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and end is not set, it would mean everything at, or after, 1455309628000. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact. - * - */ - public PathSpec end() { - return new PathSpec(getPathComponents(), "end"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(3); - } - - /** - * Represents the inclusive (greater than or equal to) value in which to start the range. This field is optional. An unset field here indicates an open range; for example, if end is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and start is not set, it would indicate times up to, but excluding, 1455309628000. Note that this interpretation was not originally documented. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact. - * - */ - public DateTimeInterval.ProjectionMask withStart() { - getDataMap().put("start", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Represents the exclusive (strictly less than) value in which to end the range. This field is optional. An unset field here indicates an open range; for example, if start is 1455309628000 (Fri, 12 Feb 2016 20:40:28 GMT), and end is not set, it would mean everything at, or after, 1455309628000. New uses of this model should follow this interpretation, but older models may not, and their documentation should reflect this fact. - * - */ - public DateTimeInterval.ProjectionMask withEnd() { - getDataMap().put("end", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Dimension.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Dimension.java deleted file mode 100644 index 6712e608c..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Dimension.java +++ /dev/null @@ -1,390 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Dimension.pdl.") -public class Dimension - extends RecordTemplate -{ - - private final static Dimension.Fields _fields = new Dimension.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}", SchemaFormatType.PDL)); - private DimensionType _typeField = null; - private Integer _shapeField = null; - private Dimension.ChangeListener __changeListener = new Dimension.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Type = SCHEMA.getField("type"); - private final static RecordDataSchema.Field FIELD_Shape = SCHEMA.getField("shape"); - - public Dimension() { - super(new DataMap(3, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public Dimension(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static Dimension.Fields fields() { - return _fields; - } - - public static Dimension.ProjectionMask createMask() { - return new Dimension.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for type - * - * @see Dimension.Fields#type - */ - public boolean hasType() { - if (_typeField!= null) { - return true; - } - return super._map.containsKey("type"); - } - - /** - * Remover for type - * - * @see Dimension.Fields#type - */ - public void removeType() { - super._map.remove("type"); - } - - /** - * Getter for type - * - * @see Dimension.Fields#type - */ - public DimensionType getType(GetMode mode) { - switch (mode) { - case STRICT: - return getType(); - case DEFAULT: - case NULL: - if (_typeField!= null) { - return _typeField; - } else { - Object __rawValue = super._map.get("type"); - _typeField = DataTemplateUtil.coerceEnumOutput(__rawValue, DimensionType.class, DimensionType.$UNKNOWN); - return _typeField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for type - * - * @return - * Required field. Could be null for partial record. - * @see Dimension.Fields#type - */ - @Nonnull - public DimensionType getType() { - if (_typeField!= null) { - return _typeField; - } else { - Object __rawValue = super._map.get("type"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("type"); - } - _typeField = DataTemplateUtil.coerceEnumOutput(__rawValue, DimensionType.class, DimensionType.$UNKNOWN); - return _typeField; - } - } - - /** - * Setter for type - * - * @see Dimension.Fields#type - */ - public Dimension setType(DimensionType value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setType(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field type of com.linkedin.feathr.compute.Dimension"); - } else { - CheckedUtil.putWithoutChecking(super._map, "type", value.name()); - _typeField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeType(); - } else { - CheckedUtil.putWithoutChecking(super._map, "type", value.name()); - _typeField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "type", value.name()); - _typeField = value; - } - break; - } - return this; - } - - /** - * Setter for type - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Dimension.Fields#type - */ - public Dimension setType( - @Nonnull - DimensionType value) { - if (value == null) { - throw new NullPointerException("Cannot set field type of com.linkedin.feathr.compute.Dimension to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "type", value.name()); - _typeField = value; - } - return this; - } - - /** - * Existence checker for shape - * - * @see Dimension.Fields#shape - */ - public boolean hasShape() { - if (_shapeField!= null) { - return true; - } - return super._map.containsKey("shape"); - } - - /** - * Remover for shape - * - * @see Dimension.Fields#shape - */ - public void removeShape() { - super._map.remove("shape"); - } - - /** - * Getter for shape - * - * @see Dimension.Fields#shape - */ - public Integer getShape(GetMode mode) { - return getShape(); - } - - /** - * Getter for shape - * - * @return - * Optional field. Always check for null. - * @see Dimension.Fields#shape - */ - @Nullable - public Integer getShape() { - if (_shapeField!= null) { - return _shapeField; - } else { - Object __rawValue = super._map.get("shape"); - _shapeField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _shapeField; - } - } - - /** - * Setter for shape - * - * @see Dimension.Fields#shape - */ - public Dimension setShape(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setShape(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeShape(); - } else { - CheckedUtil.putWithoutChecking(super._map, "shape", DataTemplateUtil.coerceIntInput(value)); - _shapeField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "shape", DataTemplateUtil.coerceIntInput(value)); - _shapeField = value; - } - break; - } - return this; - } - - /** - * Setter for shape - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Dimension.Fields#shape - */ - public Dimension setShape( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field shape of com.linkedin.feathr.compute.Dimension to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "shape", DataTemplateUtil.coerceIntInput(value)); - _shapeField = value; - } - return this; - } - - /** - * Setter for shape - * - * @see Dimension.Fields#shape - */ - public Dimension setShape(int value) { - CheckedUtil.putWithoutChecking(super._map, "shape", DataTemplateUtil.coerceIntInput(value)); - _shapeField = value; - return this; - } - - @Override - public Dimension clone() - throws CloneNotSupportedException - { - Dimension __clone = ((Dimension) super.clone()); - __clone.__changeListener = new Dimension.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public Dimension copy() - throws CloneNotSupportedException - { - Dimension __copy = ((Dimension) super.copy()); - __copy._shapeField = null; - __copy._typeField = null; - __copy.__changeListener = new Dimension.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final Dimension __objectRef; - - private ChangeListener(Dimension reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "shape": - __objectRef._shapeField = null; - break; - case "type": - __objectRef._typeField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Type of the dimension in the tensor. Each dimension can have a different type. - * - */ - public PathSpec type() { - return new PathSpec(getPathComponents(), "type"); - } - - /** - * Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime. - * - */ - public PathSpec shape() { - return new PathSpec(getPathComponents(), "shape"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(3); - } - - /** - * Type of the dimension in the tensor. Each dimension can have a different type. - * - */ - public Dimension.ProjectionMask withType() { - getDataMap().put("type", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime. - * - */ - public Dimension.ProjectionMask withShape() { - getDataMap().put("shape", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionArray.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionArray.java deleted file mode 100644 index 78c173375..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionArray.java +++ /dev/null @@ -1,118 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.schema.ArrayDataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.TemplateOutputCastException; -import com.linkedin.data.template.WrappingArrayTemplate; - -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\TensorFeatureFormat.pdl.") -public class DimensionArray - extends WrappingArrayTemplate -{ - - private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[{namespace com.linkedin.feathr.compute/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}}]", SchemaFormatType.PDL)); - - public DimensionArray() { - this(new DataList()); - } - - public DimensionArray(int initialCapacity) { - this(new DataList(initialCapacity)); - } - - public DimensionArray(Collection c) { - this(new DataList(c.size())); - addAll(c); - } - - public DimensionArray(DataList data) { - super(data, SCHEMA, Dimension.class); - } - - public DimensionArray(Dimension first, Dimension... rest) { - this(new DataList((rest.length + 1))); - add(first); - addAll(Arrays.asList(rest)); - } - - public static ArrayDataSchema dataSchema() { - return SCHEMA; - } - - public static DimensionArray.ProjectionMask createMask() { - return new DimensionArray.ProjectionMask(); - } - - @Override - public DimensionArray clone() - throws CloneNotSupportedException - { - DimensionArray __clone = ((DimensionArray) super.clone()); - return __clone; - } - - @Override - public DimensionArray copy() - throws CloneNotSupportedException - { - DimensionArray __copy = ((DimensionArray) super.copy()); - return __copy; - } - - @Override - protected Dimension coerceOutput(Object object) - throws TemplateOutputCastException - { - assert(object != null); - return ((object == null)?null:new Dimension(DataTemplateUtil.castOrThrow(object, DataMap.class))); - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.compute.Dimension.Fields items() { - return new com.linkedin.feathr.compute.Dimension.Fields(getPathComponents(), PathSpec.WILDCARD); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.Dimension.ProjectionMask _itemsMask; - - ProjectionMask() { - super(4); - } - - public DimensionArray.ProjectionMask withItems(Function nestedMask) { - _itemsMask = nestedMask.apply(((_itemsMask == null)?Dimension.createMask():_itemsMask)); - getDataMap().put("$*", _itemsMask.getDataMap()); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionType.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionType.java deleted file mode 100644 index d996b4dcc..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/DimensionType.java +++ /dev/null @@ -1,48 +0,0 @@ - -package com.linkedin.feathr.compute; - -import javax.annotation.Generated; -import com.linkedin.data.schema.EnumDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; - - -/** - * Supported dimension types for tensors in Quince and feathr. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\DimensionType.pdl.") -public enum DimensionType { - - - /** - * Long. - * - */ - LONG, - - /** - * Integer. - * - */ - INT, - - /** - * String. - * - */ - STRING, - - /** - * Boolean. - * - */ - BOOLEAN, - $UNKNOWN; - private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}", SchemaFormatType.PDL)); - - public static EnumDataSchema dataSchema() { - return SCHEMA; - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/External.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/External.java deleted file mode 100644 index 1d956ea27..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/External.java +++ /dev/null @@ -1,557 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * A temporary node which would exist only while parsing the graph. For example, when parsing an object if there is a reference to a feature - * name, we will create an external node. This would get resolved later in the computation. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\External.pdl.") -public class External - extends RecordTemplate -{ - - private final static External.Fields _fields = new External.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**A temporary node which would exist only while parsing the graph. For example, when parsing an object if there is a reference to a feature\r\nname, we will create an external node. This would get resolved later in the computation.*/record External includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**Name of the external object it should refer to.*/name:string}", SchemaFormatType.PDL)); - private Integer _idField = null; - private ConcreteKey _concreteKeyField = null; - private String _nameField = null; - private External.ChangeListener __changeListener = new External.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); - private final static RecordDataSchema.Field FIELD_ConcreteKey = SCHEMA.getField("concreteKey"); - private final static RecordDataSchema.Field FIELD_Name = SCHEMA.getField("name"); - - public External() { - super(new DataMap(4, 0.75F), SCHEMA, 2); - addChangeListener(__changeListener); - } - - public External(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static External.Fields fields() { - return _fields; - } - - public static External.ProjectionMask createMask() { - return new External.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for id - * - * @see External.Fields#id - */ - public boolean hasId() { - if (_idField!= null) { - return true; - } - return super._map.containsKey("id"); - } - - /** - * Remover for id - * - * @see External.Fields#id - */ - public void removeId() { - super._map.remove("id"); - } - - /** - * Getter for id - * - * @see External.Fields#id - */ - public Integer getId(GetMode mode) { - switch (mode) { - case STRICT: - return getId(); - case DEFAULT: - case NULL: - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for id - * - * @return - * Required field. Could be null for partial record. - * @see External.Fields#id - */ - @Nonnull - public Integer getId() { - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("id"); - } - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - - /** - * Setter for id - * - * @see External.Fields#id - */ - public External setId(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setId(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.External"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeId(); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - } - return this; - } - - /** - * Setter for id - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see External.Fields#id - */ - public External setId( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.External to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - return this; - } - - /** - * Setter for id - * - * @see External.Fields#id - */ - public External setId(int value) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - return this; - } - - /** - * Existence checker for concreteKey - * - * @see External.Fields#concreteKey - */ - public boolean hasConcreteKey() { - if (_concreteKeyField!= null) { - return true; - } - return super._map.containsKey("concreteKey"); - } - - /** - * Remover for concreteKey - * - * @see External.Fields#concreteKey - */ - public void removeConcreteKey() { - super._map.remove("concreteKey"); - } - - /** - * Getter for concreteKey - * - * @see External.Fields#concreteKey - */ - public ConcreteKey getConcreteKey(GetMode mode) { - return getConcreteKey(); - } - - /** - * Getter for concreteKey - * - * @return - * Optional field. Always check for null. - * @see External.Fields#concreteKey - */ - @Nullable - public ConcreteKey getConcreteKey() { - if (_concreteKeyField!= null) { - return _concreteKeyField; - } else { - Object __rawValue = super._map.get("concreteKey"); - _concreteKeyField = ((__rawValue == null)?null:new ConcreteKey(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _concreteKeyField; - } - } - - /** - * Setter for concreteKey - * - * @see External.Fields#concreteKey - */ - public External setConcreteKey(ConcreteKey value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setConcreteKey(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeConcreteKey(); - } else { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - break; - } - return this; - } - - /** - * Setter for concreteKey - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see External.Fields#concreteKey - */ - public External setConcreteKey( - @Nonnull - ConcreteKey value) { - if (value == null) { - throw new NullPointerException("Cannot set field concreteKey of com.linkedin.feathr.compute.External to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - return this; - } - - /** - * Existence checker for name - * - * @see External.Fields#name - */ - public boolean hasName() { - if (_nameField!= null) { - return true; - } - return super._map.containsKey("name"); - } - - /** - * Remover for name - * - * @see External.Fields#name - */ - public void removeName() { - super._map.remove("name"); - } - - /** - * Getter for name - * - * @see External.Fields#name - */ - public String getName(GetMode mode) { - switch (mode) { - case STRICT: - return getName(); - case DEFAULT: - case NULL: - if (_nameField!= null) { - return _nameField; - } else { - Object __rawValue = super._map.get("name"); - _nameField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _nameField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for name - * - * @return - * Required field. Could be null for partial record. - * @see External.Fields#name - */ - @Nonnull - public String getName() { - if (_nameField!= null) { - return _nameField; - } else { - Object __rawValue = super._map.get("name"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("name"); - } - _nameField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _nameField; - } - } - - /** - * Setter for name - * - * @see External.Fields#name - */ - public External setName(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setName(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field name of com.linkedin.feathr.compute.External"); - } else { - CheckedUtil.putWithoutChecking(super._map, "name", value); - _nameField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeName(); - } else { - CheckedUtil.putWithoutChecking(super._map, "name", value); - _nameField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "name", value); - _nameField = value; - } - break; - } - return this; - } - - /** - * Setter for name - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see External.Fields#name - */ - public External setName( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field name of com.linkedin.feathr.compute.External to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "name", value); - _nameField = value; - } - return this; - } - - @Override - public External clone() - throws CloneNotSupportedException - { - External __clone = ((External) super.clone()); - __clone.__changeListener = new External.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public External copy() - throws CloneNotSupportedException - { - External __copy = ((External) super.copy()); - __copy._nameField = null; - __copy._idField = null; - __copy._concreteKeyField = null; - __copy.__changeListener = new External.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final External __objectRef; - - private ChangeListener(External reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "name": - __objectRef._nameField = null; - break; - case "id": - __objectRef._idField = null; - break; - case "concreteKey": - __objectRef._concreteKeyField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The node would be represented by this id. - * - */ - public PathSpec id() { - return new PathSpec(getPathComponents(), "id"); - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public com.linkedin.feathr.compute.ConcreteKey.Fields concreteKey() { - return new com.linkedin.feathr.compute.ConcreteKey.Fields(getPathComponents(), "concreteKey"); - } - - /** - * Name of the external object it should refer to. - * - */ - public PathSpec name() { - return new PathSpec(getPathComponents(), "name"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.ConcreteKey.ProjectionMask _concreteKeyMask; - - ProjectionMask() { - super(4); - } - - /** - * The node would be represented by this id. - * - */ - public External.ProjectionMask withId() { - getDataMap().put("id", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public External.ProjectionMask withConcreteKey(Function nestedMask) { - _concreteKeyMask = nestedMask.apply(((_concreteKeyMask == null)?ConcreteKey.createMask():_concreteKeyMask)); - getDataMap().put("concreteKey", _concreteKeyMask.getDataMap()); - return this; - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public External.ProjectionMask withConcreteKey() { - _concreteKeyMask = null; - getDataMap().put("concreteKey", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Name of the external object it should refer to. - * - */ - public External.ProjectionMask withName() { - getDataMap().put("name", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureValue.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureValue.java deleted file mode 100644 index 24f400003..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureValue.java +++ /dev/null @@ -1,431 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import javax.annotation.Generated; -import com.linkedin.data.ByteString; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.DataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.TyperefDataSchema; -import com.linkedin.data.schema.UnionDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.HasTyperefInfo; -import com.linkedin.data.template.TyperefInfo; -import com.linkedin.data.template.UnionTemplate; - -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\FeatureValue.pdl.") -public class FeatureValue - extends UnionTemplate - implements HasTyperefInfo -{ - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[boolean,int,long,float,double,string,bytes]", SchemaFormatType.PDL)); - private java.lang.Boolean _booleanMember = null; - private Integer _intMember = null; - private java.lang.Long _longMember = null; - private java.lang.Float _floatMember = null; - private java.lang.Double _doubleMember = null; - private java.lang.String _stringMember = null; - private ByteString _bytesMember = null; - private FeatureValue.ChangeListener __changeListener = new FeatureValue.ChangeListener(this); - private final static DataSchema MEMBER_Boolean = SCHEMA.getTypeByMemberKey("boolean"); - private final static DataSchema MEMBER_Int = SCHEMA.getTypeByMemberKey("int"); - private final static DataSchema MEMBER_Long = SCHEMA.getTypeByMemberKey("long"); - private final static DataSchema MEMBER_Float = SCHEMA.getTypeByMemberKey("float"); - private final static DataSchema MEMBER_Double = SCHEMA.getTypeByMemberKey("double"); - private final static DataSchema MEMBER_String = SCHEMA.getTypeByMemberKey("string"); - private final static DataSchema MEMBER_Bytes = SCHEMA.getTypeByMemberKey("bytes"); - private final static TyperefInfo TYPEREFINFO = new FeatureValue.UnionTyperefInfo(); - - public FeatureValue() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public FeatureValue(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static FeatureValue create(java.lang.Boolean value) { - FeatureValue newUnion = new FeatureValue(); - newUnion.setBoolean(value); - return newUnion; - } - - public boolean isBoolean() { - return memberIs("boolean"); - } - - public java.lang.Boolean getBoolean() { - checkNotNull(); - if (_booleanMember!= null) { - return _booleanMember; - } - Object __rawValue = super._map.get("boolean"); - _booleanMember = DataTemplateUtil.coerceBooleanOutput(__rawValue); - return _booleanMember; - } - - public void setBoolean(java.lang.Boolean value) { - checkNotNull(); - super._map.clear(); - _booleanMember = value; - CheckedUtil.putWithoutChecking(super._map, "boolean", value); - } - - public static FeatureValue create(Integer value) { - FeatureValue newUnion = new FeatureValue(); - newUnion.setInt(value); - return newUnion; - } - - public boolean isInt() { - return memberIs("int"); - } - - public Integer getInt() { - checkNotNull(); - if (_intMember!= null) { - return _intMember; - } - Object __rawValue = super._map.get("int"); - _intMember = DataTemplateUtil.coerceIntOutput(__rawValue); - return _intMember; - } - - public void setInt(Integer value) { - checkNotNull(); - super._map.clear(); - _intMember = value; - CheckedUtil.putWithoutChecking(super._map, "int", DataTemplateUtil.coerceIntInput(value)); - } - - public static FeatureValue create(java.lang.Long value) { - FeatureValue newUnion = new FeatureValue(); - newUnion.setLong(value); - return newUnion; - } - - public boolean isLong() { - return memberIs("long"); - } - - public java.lang.Long getLong() { - checkNotNull(); - if (_longMember!= null) { - return _longMember; - } - Object __rawValue = super._map.get("long"); - _longMember = DataTemplateUtil.coerceLongOutput(__rawValue); - return _longMember; - } - - public void setLong(java.lang.Long value) { - checkNotNull(); - super._map.clear(); - _longMember = value; - CheckedUtil.putWithoutChecking(super._map, "long", DataTemplateUtil.coerceLongInput(value)); - } - - public static FeatureValue create(java.lang.Float value) { - FeatureValue newUnion = new FeatureValue(); - newUnion.setFloat(value); - return newUnion; - } - - public boolean isFloat() { - return memberIs("float"); - } - - public java.lang.Float getFloat() { - checkNotNull(); - if (_floatMember!= null) { - return _floatMember; - } - Object __rawValue = super._map.get("float"); - _floatMember = DataTemplateUtil.coerceFloatOutput(__rawValue); - return _floatMember; - } - - public void setFloat(java.lang.Float value) { - checkNotNull(); - super._map.clear(); - _floatMember = value; - CheckedUtil.putWithoutChecking(super._map, "float", DataTemplateUtil.coerceFloatInput(value)); - } - - public static FeatureValue create(java.lang.Double value) { - FeatureValue newUnion = new FeatureValue(); - newUnion.setDouble(value); - return newUnion; - } - - public boolean isDouble() { - return memberIs("double"); - } - - public java.lang.Double getDouble() { - checkNotNull(); - if (_doubleMember!= null) { - return _doubleMember; - } - Object __rawValue = super._map.get("double"); - _doubleMember = DataTemplateUtil.coerceDoubleOutput(__rawValue); - return _doubleMember; - } - - public void setDouble(java.lang.Double value) { - checkNotNull(); - super._map.clear(); - _doubleMember = value; - CheckedUtil.putWithoutChecking(super._map, "double", DataTemplateUtil.coerceDoubleInput(value)); - } - - public static FeatureValue create(java.lang.String value) { - FeatureValue newUnion = new FeatureValue(); - newUnion.setString(value); - return newUnion; - } - - public boolean isString() { - return memberIs("string"); - } - - public java.lang.String getString() { - checkNotNull(); - if (_stringMember!= null) { - return _stringMember; - } - Object __rawValue = super._map.get("string"); - _stringMember = DataTemplateUtil.coerceStringOutput(__rawValue); - return _stringMember; - } - - public void setString(java.lang.String value) { - checkNotNull(); - super._map.clear(); - _stringMember = value; - CheckedUtil.putWithoutChecking(super._map, "string", value); - } - - public static FeatureValue create(ByteString value) { - FeatureValue newUnion = new FeatureValue(); - newUnion.setBytes(value); - return newUnion; - } - - public boolean isBytes() { - return memberIs("bytes"); - } - - public ByteString getBytes() { - checkNotNull(); - if (_bytesMember!= null) { - return _bytesMember; - } - Object __rawValue = super._map.get("bytes"); - _bytesMember = DataTemplateUtil.coerceBytesOutput(__rawValue); - return _bytesMember; - } - - public void setBytes(ByteString value) { - checkNotNull(); - super._map.clear(); - _bytesMember = value; - CheckedUtil.putWithoutChecking(super._map, "bytes", value); - } - - public static FeatureValue.ProjectionMask createMask() { - return new FeatureValue.ProjectionMask(); - } - - @Override - public FeatureValue clone() - throws CloneNotSupportedException - { - FeatureValue __clone = ((FeatureValue) super.clone()); - __clone.__changeListener = new FeatureValue.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public FeatureValue copy() - throws CloneNotSupportedException - { - FeatureValue __copy = ((FeatureValue) super.copy()); - __copy._booleanMember = null; - __copy._stringMember = null; - __copy._doubleMember = null; - __copy._bytesMember = null; - __copy._floatMember = null; - __copy._intMember = null; - __copy._longMember = null; - __copy.__changeListener = new FeatureValue.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - public TyperefInfo typerefInfo() { - return TYPEREFINFO; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final FeatureValue __objectRef; - - private ChangeListener(FeatureValue reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(java.lang.String key, Object value) { - switch (key) { - case "boolean": - __objectRef._booleanMember = null; - break; - case "string": - __objectRef._stringMember = null; - break; - case "double": - __objectRef._doubleMember = null; - break; - case "bytes": - __objectRef._bytesMember = null; - break; - case "float": - __objectRef._floatMember = null; - break; - case "int": - __objectRef._intMember = null; - break; - case "long": - __objectRef._longMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, java.lang.String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public PathSpec Boolean() { - return new PathSpec(getPathComponents(), "boolean"); - } - - public PathSpec Int() { - return new PathSpec(getPathComponents(), "int"); - } - - public PathSpec Long() { - return new PathSpec(getPathComponents(), "long"); - } - - public PathSpec Float() { - return new PathSpec(getPathComponents(), "float"); - } - - public PathSpec Double() { - return new PathSpec(getPathComponents(), "double"); - } - - public PathSpec String() { - return new PathSpec(getPathComponents(), "string"); - } - - public PathSpec Bytes() { - return new PathSpec(getPathComponents(), "bytes"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(10); - } - - public FeatureValue.ProjectionMask withBoolean() { - getDataMap().put("boolean", MaskMap.POSITIVE_MASK); - return this; - } - - public FeatureValue.ProjectionMask withInt() { - getDataMap().put("int", MaskMap.POSITIVE_MASK); - return this; - } - - public FeatureValue.ProjectionMask withLong() { - getDataMap().put("long", MaskMap.POSITIVE_MASK); - return this; - } - - public FeatureValue.ProjectionMask withFloat() { - getDataMap().put("float", MaskMap.POSITIVE_MASK); - return this; - } - - public FeatureValue.ProjectionMask withDouble() { - getDataMap().put("double", MaskMap.POSITIVE_MASK); - return this; - } - - public FeatureValue.ProjectionMask withString() { - getDataMap().put("string", MaskMap.POSITIVE_MASK); - return this; - } - - public FeatureValue.ProjectionMask withBytes() { - getDataMap().put("bytes", MaskMap.POSITIVE_MASK); - return this; - } - - } - - - /** - * Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases. - * - */ - private final static class UnionTyperefInfo - extends TyperefInfo - { - - private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]", SchemaFormatType.PDL)); - - public UnionTyperefInfo() { - super(SCHEMA); - } - - public static TyperefDataSchema dataSchema() { - return SCHEMA; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureVersion.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureVersion.java deleted file mode 100644 index 432fa6410..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FeatureVersion.java +++ /dev/null @@ -1,526 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.SetMode; - - -/** - * - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\FeatureVersion.pdl.") -public class FeatureVersion - extends RecordTemplate -{ - - private final static FeatureVersion.Fields _fields = new FeatureVersion.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute,record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}", SchemaFormatType.PDL)); - private FrameFeatureType _typeField = null; - private TensorFeatureFormat _formatField = null; - private FeatureValue _defaultValueField = null; - private FeatureVersion.ChangeListener __changeListener = new FeatureVersion.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Type = SCHEMA.getField("type"); - private final static FrameFeatureType DEFAULT_Type; - private final static RecordDataSchema.Field FIELD_Format = SCHEMA.getField("format"); - private final static RecordDataSchema.Field FIELD_DefaultValue = SCHEMA.getField("defaultValue"); - - static { - DEFAULT_Type = DataTemplateUtil.coerceEnumOutput(FIELD_Type.getDefault(), FrameFeatureType.class, FrameFeatureType.$UNKNOWN); - } - - public FeatureVersion() { - super(new DataMap(4, 0.75F), SCHEMA, 3); - addChangeListener(__changeListener); - } - - public FeatureVersion(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static FeatureVersion.Fields fields() { - return _fields; - } - - public static FeatureVersion.ProjectionMask createMask() { - return new FeatureVersion.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for type - * - * @see FeatureVersion.Fields#type - */ - public boolean hasType() { - if (_typeField!= null) { - return true; - } - return super._map.containsKey("type"); - } - - /** - * Remover for type - * - * @see FeatureVersion.Fields#type - */ - public void removeType() { - super._map.remove("type"); - } - - /** - * Getter for type - * - * @see FeatureVersion.Fields#type - */ - public FrameFeatureType getType(GetMode mode) { - switch (mode) { - case STRICT: - case DEFAULT: - return getType(); - case NULL: - if (_typeField!= null) { - return _typeField; - } else { - Object __rawValue = super._map.get("type"); - _typeField = DataTemplateUtil.coerceEnumOutput(__rawValue, FrameFeatureType.class, FrameFeatureType.$UNKNOWN); - return _typeField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for type - * - * @return - * Required field. Could be null for partial record. - * @see FeatureVersion.Fields#type - */ - @Nonnull - public FrameFeatureType getType() { - if (_typeField!= null) { - return _typeField; - } else { - Object __rawValue = super._map.get("type"); - if (__rawValue == null) { - return DEFAULT_Type; - } - _typeField = DataTemplateUtil.coerceEnumOutput(__rawValue, FrameFeatureType.class, FrameFeatureType.$UNKNOWN); - return _typeField; - } - } - - /** - * Setter for type - * - * @see FeatureVersion.Fields#type - */ - public FeatureVersion setType(FrameFeatureType value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setType(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field type of com.linkedin.feathr.compute.FeatureVersion"); - } else { - CheckedUtil.putWithoutChecking(super._map, "type", value.name()); - _typeField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeType(); - } else { - CheckedUtil.putWithoutChecking(super._map, "type", value.name()); - _typeField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "type", value.name()); - _typeField = value; - } - break; - } - return this; - } - - /** - * Setter for type - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see FeatureVersion.Fields#type - */ - public FeatureVersion setType( - @Nonnull - FrameFeatureType value) { - if (value == null) { - throw new NullPointerException("Cannot set field type of com.linkedin.feathr.compute.FeatureVersion to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "type", value.name()); - _typeField = value; - } - return this; - } - - /** - * Existence checker for format - * - * @see FeatureVersion.Fields#format - */ - public boolean hasFormat() { - if (_formatField!= null) { - return true; - } - return super._map.containsKey("format"); - } - - /** - * Remover for format - * - * @see FeatureVersion.Fields#format - */ - public void removeFormat() { - super._map.remove("format"); - } - - /** - * Getter for format - * - * @see FeatureVersion.Fields#format - */ - public TensorFeatureFormat getFormat(GetMode mode) { - return getFormat(); - } - - /** - * Getter for format - * - * @return - * Optional field. Always check for null. - * @see FeatureVersion.Fields#format - */ - @Nullable - public TensorFeatureFormat getFormat() { - if (_formatField!= null) { - return _formatField; - } else { - Object __rawValue = super._map.get("format"); - _formatField = ((__rawValue == null)?null:new TensorFeatureFormat(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _formatField; - } - } - - /** - * Setter for format - * - * @see FeatureVersion.Fields#format - */ - public FeatureVersion setFormat(TensorFeatureFormat value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFormat(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeFormat(); - } else { - CheckedUtil.putWithoutChecking(super._map, "format", value.data()); - _formatField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "format", value.data()); - _formatField = value; - } - break; - } - return this; - } - - /** - * Setter for format - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see FeatureVersion.Fields#format - */ - public FeatureVersion setFormat( - @Nonnull - TensorFeatureFormat value) { - if (value == null) { - throw new NullPointerException("Cannot set field format of com.linkedin.feathr.compute.FeatureVersion to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "format", value.data()); - _formatField = value; - } - return this; - } - - /** - * Existence checker for defaultValue - * - * @see FeatureVersion.Fields#defaultValue - */ - public boolean hasDefaultValue() { - if (_defaultValueField!= null) { - return true; - } - return super._map.containsKey("defaultValue"); - } - - /** - * Remover for defaultValue - * - * @see FeatureVersion.Fields#defaultValue - */ - public void removeDefaultValue() { - super._map.remove("defaultValue"); - } - - /** - * Getter for defaultValue - * - * @see FeatureVersion.Fields#defaultValue - */ - public FeatureValue getDefaultValue(GetMode mode) { - return getDefaultValue(); - } - - /** - * Getter for defaultValue - * - * @return - * Optional field. Always check for null. - * @see FeatureVersion.Fields#defaultValue - */ - @Nullable - public FeatureValue getDefaultValue() { - if (_defaultValueField!= null) { - return _defaultValueField; - } else { - Object __rawValue = super._map.get("defaultValue"); - _defaultValueField = ((__rawValue == null)?null:new FeatureValue(__rawValue)); - return _defaultValueField; - } - } - - /** - * Setter for defaultValue - * - * @see FeatureVersion.Fields#defaultValue - */ - public FeatureVersion setDefaultValue(FeatureValue value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setDefaultValue(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeDefaultValue(); - } else { - CheckedUtil.putWithoutChecking(super._map, "defaultValue", value.data()); - _defaultValueField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "defaultValue", value.data()); - _defaultValueField = value; - } - break; - } - return this; - } - - /** - * Setter for defaultValue - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see FeatureVersion.Fields#defaultValue - */ - public FeatureVersion setDefaultValue( - @Nonnull - FeatureValue value) { - if (value == null) { - throw new NullPointerException("Cannot set field defaultValue of com.linkedin.feathr.compute.FeatureVersion to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "defaultValue", value.data()); - _defaultValueField = value; - } - return this; - } - - @Override - public FeatureVersion clone() - throws CloneNotSupportedException - { - FeatureVersion __clone = ((FeatureVersion) super.clone()); - __clone.__changeListener = new FeatureVersion.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public FeatureVersion copy() - throws CloneNotSupportedException - { - FeatureVersion __copy = ((FeatureVersion) super.copy()); - __copy._defaultValueField = null; - __copy._formatField = null; - __copy._typeField = null; - __copy.__changeListener = new FeatureVersion.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final FeatureVersion __objectRef; - - private ChangeListener(FeatureVersion reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "defaultValue": - __objectRef._defaultValueField = null; - break; - case "format": - __objectRef._formatField = null; - break; - case "type": - __objectRef._typeField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed - * - */ - public PathSpec type() { - return new PathSpec(getPathComponents(), "type"); - } - - /** - * Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features. - * - */ - public com.linkedin.feathr.compute.TensorFeatureFormat.Fields format() { - return new com.linkedin.feathr.compute.TensorFeatureFormat.Fields(getPathComponents(), "format"); - } - - /** - * An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data. - * - */ - public com.linkedin.feathr.compute.FeatureValue.Fields defaultValue() { - return new com.linkedin.feathr.compute.FeatureValue.Fields(getPathComponents(), "defaultValue"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.TensorFeatureFormat.ProjectionMask _formatMask; - private com.linkedin.feathr.compute.FeatureValue.ProjectionMask _defaultValueMask; - - ProjectionMask() { - super(4); - } - - /** - * Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed - * - */ - public FeatureVersion.ProjectionMask withType() { - getDataMap().put("type", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features. - * - */ - public FeatureVersion.ProjectionMask withFormat(Function nestedMask) { - _formatMask = nestedMask.apply(((_formatMask == null)?TensorFeatureFormat.createMask():_formatMask)); - getDataMap().put("format", _formatMask.getDataMap()); - return this; - } - - /** - * Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features. - * - */ - public FeatureVersion.ProjectionMask withFormat() { - _formatMask = null; - getDataMap().put("format", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data. - * - */ - public FeatureVersion.ProjectionMask withDefaultValue(Function nestedMask) { - _defaultValueMask = nestedMask.apply(((_defaultValueMask == null)?FeatureValue.createMask():_defaultValueMask)); - getDataMap().put("defaultValue", _defaultValueMask.getDataMap()); - return this; - } - - /** - * An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data. - * - */ - public FeatureVersion.ProjectionMask withDefaultValue() { - _defaultValueMask = null; - getDataMap().put("defaultValue", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FrameFeatureType.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FrameFeatureType.java deleted file mode 100644 index f92056e89..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/FrameFeatureType.java +++ /dev/null @@ -1,72 +0,0 @@ - -package com.linkedin.feathr.compute; - -import javax.annotation.Generated; -import com.linkedin.data.schema.EnumDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; - - -/** - * The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\FrameFeatureType.pdl.") -public enum FrameFeatureType { - - - /** - * Boolean valued feature - * - */ - BOOLEAN, - - /** - * Numerically valued feature such as INT, LONG, DOUBLE, etc - * - */ - NUMERIC, - - /** - * Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) - * - */ - CATEGORICAL, - - /** - * Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) - * - */ - CATEGORICAL_SET, - - /** - * Represents a feature in vector format where the the majority of the elements are non-zero - * - */ - DENSE_VECTOR, - - /** - * Represents features that has string terms and numeric value - * - */ - TERM_VECTOR, - - /** - * Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML). - * - */ - TENSOR, - - /** - * Placeholder for when no types are specified - * - */ - UNSPECIFIED, - $UNKNOWN; - private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}", SchemaFormatType.PDL)); - - public static EnumDataSchema dataSchema() { - return SCHEMA; - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyExpressionType.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyExpressionType.java deleted file mode 100644 index a8bbfd957..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyExpressionType.java +++ /dev/null @@ -1,44 +0,0 @@ - -package com.linkedin.feathr.compute; - -import javax.annotation.Generated; -import com.linkedin.data.schema.EnumDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; - - -/** - * Different key formats supported. - * Todo - We probably do not want to generalize this as a kind of key-operator in the core compute model, - * with instances such as for MVEL or SQL being available (e.g. via an OperatorId reference). - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\KeyExpressionType.pdl.") -public enum KeyExpressionType { - - - /** - * Java-based MVEL - * - */ - MVEL, - - /** - * Spark-SQL - * - */ - SQL, - - /** - * Custom java/scala UDF - * - */ - UDF, - $UNKNOWN; - private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Different key formats supported.\r\nTodo - We probably do not want to generalize this as a kind of key-operator in the core compute model,\r\nwith instances such as for MVEL or SQL being available (e.g. via an OperatorId reference).*/enum KeyExpressionType{/**Java-based MVEL*/MVEL/**Spark-SQL*/SQL/**Custom java/scala UDF*/UDF}", SchemaFormatType.PDL)); - - public static EnumDataSchema dataSchema() { - return SCHEMA; - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReference.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReference.java deleted file mode 100644 index 887c4aac8..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReference.java +++ /dev/null @@ -1,272 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * This represents the position of the key in the node which is being referred to. For example, if the original node has a key - * like [x, y], and the keyReference says 1, it is referring to y. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\KeyReference.pdl.") -public class KeyReference - extends RecordTemplate -{ - - private final static KeyReference.Fields _fields = new KeyReference.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}", SchemaFormatType.PDL)); - private Integer _positionField = null; - private KeyReference.ChangeListener __changeListener = new KeyReference.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Position = SCHEMA.getField("position"); - - public KeyReference() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public KeyReference(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static KeyReference.Fields fields() { - return _fields; - } - - public static KeyReference.ProjectionMask createMask() { - return new KeyReference.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for position - * - * @see KeyReference.Fields#position - */ - public boolean hasPosition() { - if (_positionField!= null) { - return true; - } - return super._map.containsKey("position"); - } - - /** - * Remover for position - * - * @see KeyReference.Fields#position - */ - public void removePosition() { - super._map.remove("position"); - } - - /** - * Getter for position - * - * @see KeyReference.Fields#position - */ - public Integer getPosition(GetMode mode) { - switch (mode) { - case STRICT: - return getPosition(); - case DEFAULT: - case NULL: - if (_positionField!= null) { - return _positionField; - } else { - Object __rawValue = super._map.get("position"); - _positionField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _positionField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for position - * - * @return - * Required field. Could be null for partial record. - * @see KeyReference.Fields#position - */ - @Nonnull - public Integer getPosition() { - if (_positionField!= null) { - return _positionField; - } else { - Object __rawValue = super._map.get("position"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("position"); - } - _positionField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _positionField; - } - } - - /** - * Setter for position - * - * @see KeyReference.Fields#position - */ - public KeyReference setPosition(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setPosition(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field position of com.linkedin.feathr.compute.KeyReference"); - } else { - CheckedUtil.putWithoutChecking(super._map, "position", DataTemplateUtil.coerceIntInput(value)); - _positionField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removePosition(); - } else { - CheckedUtil.putWithoutChecking(super._map, "position", DataTemplateUtil.coerceIntInput(value)); - _positionField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "position", DataTemplateUtil.coerceIntInput(value)); - _positionField = value; - } - break; - } - return this; - } - - /** - * Setter for position - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see KeyReference.Fields#position - */ - public KeyReference setPosition( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field position of com.linkedin.feathr.compute.KeyReference to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "position", DataTemplateUtil.coerceIntInput(value)); - _positionField = value; - } - return this; - } - - /** - * Setter for position - * - * @see KeyReference.Fields#position - */ - public KeyReference setPosition(int value) { - CheckedUtil.putWithoutChecking(super._map, "position", DataTemplateUtil.coerceIntInput(value)); - _positionField = value; - return this; - } - - @Override - public KeyReference clone() - throws CloneNotSupportedException - { - KeyReference __clone = ((KeyReference) super.clone()); - __clone.__changeListener = new KeyReference.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public KeyReference copy() - throws CloneNotSupportedException - { - KeyReference __copy = ((KeyReference) super.copy()); - __copy._positionField = null; - __copy.__changeListener = new KeyReference.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final KeyReference __objectRef; - - private ChangeListener(KeyReference reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "position": - __objectRef._positionField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Position in the original key array - * - */ - public PathSpec position() { - return new PathSpec(getPathComponents(), "position"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(2); - } - - /** - * Position in the original key array - * - */ - public KeyReference.ProjectionMask withPosition() { - getDataMap().put("position", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReferenceArray.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReferenceArray.java deleted file mode 100644 index df9bfe3c5..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/KeyReferenceArray.java +++ /dev/null @@ -1,118 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.schema.ArrayDataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.TemplateOutputCastException; -import com.linkedin.data.template.WrappingArrayTemplate; - -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\NodeReference.pdl.") -public class KeyReferenceArray - extends WrappingArrayTemplate -{ - - private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[{namespace com.linkedin.feathr.compute/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}}]", SchemaFormatType.PDL)); - - public KeyReferenceArray() { - this(new DataList()); - } - - public KeyReferenceArray(int initialCapacity) { - this(new DataList(initialCapacity)); - } - - public KeyReferenceArray(Collection c) { - this(new DataList(c.size())); - addAll(c); - } - - public KeyReferenceArray(DataList data) { - super(data, SCHEMA, KeyReference.class); - } - - public KeyReferenceArray(KeyReference first, KeyReference... rest) { - this(new DataList((rest.length + 1))); - add(first); - addAll(Arrays.asList(rest)); - } - - public static ArrayDataSchema dataSchema() { - return SCHEMA; - } - - public static KeyReferenceArray.ProjectionMask createMask() { - return new KeyReferenceArray.ProjectionMask(); - } - - @Override - public KeyReferenceArray clone() - throws CloneNotSupportedException - { - KeyReferenceArray __clone = ((KeyReferenceArray) super.clone()); - return __clone; - } - - @Override - public KeyReferenceArray copy() - throws CloneNotSupportedException - { - KeyReferenceArray __copy = ((KeyReferenceArray) super.copy()); - return __copy; - } - - @Override - protected KeyReference coerceOutput(Object object) - throws TemplateOutputCastException - { - assert(object != null); - return ((object == null)?null:new KeyReference(DataTemplateUtil.castOrThrow(object, DataMap.class))); - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.compute.KeyReference.Fields items() { - return new com.linkedin.feathr.compute.KeyReference.Fields(getPathComponents(), PathSpec.WILDCARD); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.KeyReference.ProjectionMask _itemsMask; - - ProjectionMask() { - super(4); - } - - public KeyReferenceArray.ProjectionMask withItems(Function nestedMask) { - _itemsMask = nestedMask.apply(((_itemsMask == null)?KeyReference.createMask():_itemsMask)); - getDataMap().put("$*", _itemsMask.getDataMap()); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralView.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralView.java deleted file mode 100644 index 7cf602ffa..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralView.java +++ /dev/null @@ -1,553 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.DataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.UnionDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; -import com.linkedin.data.template.UnionTemplate; - - -/** - * Lateral view is used in conjunction with table generating functions (eg. the most commonly used explode()), which typically generates zero or more output rows for each input row. A lateral view first applies the table generating function to each row of base table, and then joins resulting output rows to the input rows to form a virtual table with the supplied table alias. For more details and examples, refer to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+LateralView. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\LateralView.pdl.") -public class LateralView - extends RecordTemplate -{ - - private final static LateralView.Fields _fields = new LateralView.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Lateral view is used in conjunction with table generating functions (eg. the most commonly used explode()), which typically generates zero or more output rows for each input row. A lateral view first applies the table generating function to each row of base table, and then joins resulting output rows to the input rows to form a virtual table with the supplied table alias. For more details and examples, refer to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+LateralView.*/record LateralView{/**A table-generating function transforms a single input row to multiple output rows. For example, explode(array('A','B','C') will produce 3 one-column rows, which are row1: 'A'; row2: 'B'; row3: 'C'.*/tableGeneratingFunction:union[/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}]/**Represents the alias for referencing the generated virtual table. It will be used in subsequent statements (eg. filter, groupBy) in the sliding window feature definition.*/virtualTableAlias:string}", SchemaFormatType.PDL)); - private LateralView.TableGeneratingFunction _tableGeneratingFunctionField = null; - private String _virtualTableAliasField = null; - private LateralView.ChangeListener __changeListener = new LateralView.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_TableGeneratingFunction = SCHEMA.getField("tableGeneratingFunction"); - private final static RecordDataSchema.Field FIELD_VirtualTableAlias = SCHEMA.getField("virtualTableAlias"); - - public LateralView() { - super(new DataMap(3, 0.75F), SCHEMA, 2); - addChangeListener(__changeListener); - } - - public LateralView(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static LateralView.Fields fields() { - return _fields; - } - - public static LateralView.ProjectionMask createMask() { - return new LateralView.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for tableGeneratingFunction - * - * @see LateralView.Fields#tableGeneratingFunction - */ - public boolean hasTableGeneratingFunction() { - if (_tableGeneratingFunctionField!= null) { - return true; - } - return super._map.containsKey("tableGeneratingFunction"); - } - - /** - * Remover for tableGeneratingFunction - * - * @see LateralView.Fields#tableGeneratingFunction - */ - public void removeTableGeneratingFunction() { - super._map.remove("tableGeneratingFunction"); - } - - /** - * Getter for tableGeneratingFunction - * - * @see LateralView.Fields#tableGeneratingFunction - */ - public LateralView.TableGeneratingFunction getTableGeneratingFunction(GetMode mode) { - switch (mode) { - case STRICT: - return getTableGeneratingFunction(); - case DEFAULT: - case NULL: - if (_tableGeneratingFunctionField!= null) { - return _tableGeneratingFunctionField; - } else { - Object __rawValue = super._map.get("tableGeneratingFunction"); - _tableGeneratingFunctionField = ((__rawValue == null)?null:new LateralView.TableGeneratingFunction(__rawValue)); - return _tableGeneratingFunctionField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for tableGeneratingFunction - * - * @return - * Required field. Could be null for partial record. - * @see LateralView.Fields#tableGeneratingFunction - */ - @Nonnull - public LateralView.TableGeneratingFunction getTableGeneratingFunction() { - if (_tableGeneratingFunctionField!= null) { - return _tableGeneratingFunctionField; - } else { - Object __rawValue = super._map.get("tableGeneratingFunction"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("tableGeneratingFunction"); - } - _tableGeneratingFunctionField = ((__rawValue == null)?null:new LateralView.TableGeneratingFunction(__rawValue)); - return _tableGeneratingFunctionField; - } - } - - /** - * Setter for tableGeneratingFunction - * - * @see LateralView.Fields#tableGeneratingFunction - */ - public LateralView setTableGeneratingFunction(LateralView.TableGeneratingFunction value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setTableGeneratingFunction(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field tableGeneratingFunction of com.linkedin.feathr.compute.LateralView"); - } else { - CheckedUtil.putWithoutChecking(super._map, "tableGeneratingFunction", value.data()); - _tableGeneratingFunctionField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeTableGeneratingFunction(); - } else { - CheckedUtil.putWithoutChecking(super._map, "tableGeneratingFunction", value.data()); - _tableGeneratingFunctionField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "tableGeneratingFunction", value.data()); - _tableGeneratingFunctionField = value; - } - break; - } - return this; - } - - /** - * Setter for tableGeneratingFunction - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see LateralView.Fields#tableGeneratingFunction - */ - public LateralView setTableGeneratingFunction( - @Nonnull - LateralView.TableGeneratingFunction value) { - if (value == null) { - throw new NullPointerException("Cannot set field tableGeneratingFunction of com.linkedin.feathr.compute.LateralView to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "tableGeneratingFunction", value.data()); - _tableGeneratingFunctionField = value; - } - return this; - } - - /** - * Existence checker for virtualTableAlias - * - * @see LateralView.Fields#virtualTableAlias - */ - public boolean hasVirtualTableAlias() { - if (_virtualTableAliasField!= null) { - return true; - } - return super._map.containsKey("virtualTableAlias"); - } - - /** - * Remover for virtualTableAlias - * - * @see LateralView.Fields#virtualTableAlias - */ - public void removeVirtualTableAlias() { - super._map.remove("virtualTableAlias"); - } - - /** - * Getter for virtualTableAlias - * - * @see LateralView.Fields#virtualTableAlias - */ - public String getVirtualTableAlias(GetMode mode) { - switch (mode) { - case STRICT: - return getVirtualTableAlias(); - case DEFAULT: - case NULL: - if (_virtualTableAliasField!= null) { - return _virtualTableAliasField; - } else { - Object __rawValue = super._map.get("virtualTableAlias"); - _virtualTableAliasField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _virtualTableAliasField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for virtualTableAlias - * - * @return - * Required field. Could be null for partial record. - * @see LateralView.Fields#virtualTableAlias - */ - @Nonnull - public String getVirtualTableAlias() { - if (_virtualTableAliasField!= null) { - return _virtualTableAliasField; - } else { - Object __rawValue = super._map.get("virtualTableAlias"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("virtualTableAlias"); - } - _virtualTableAliasField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _virtualTableAliasField; - } - } - - /** - * Setter for virtualTableAlias - * - * @see LateralView.Fields#virtualTableAlias - */ - public LateralView setVirtualTableAlias(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setVirtualTableAlias(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field virtualTableAlias of com.linkedin.feathr.compute.LateralView"); - } else { - CheckedUtil.putWithoutChecking(super._map, "virtualTableAlias", value); - _virtualTableAliasField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeVirtualTableAlias(); - } else { - CheckedUtil.putWithoutChecking(super._map, "virtualTableAlias", value); - _virtualTableAliasField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "virtualTableAlias", value); - _virtualTableAliasField = value; - } - break; - } - return this; - } - - /** - * Setter for virtualTableAlias - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see LateralView.Fields#virtualTableAlias - */ - public LateralView setVirtualTableAlias( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field virtualTableAlias of com.linkedin.feathr.compute.LateralView to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "virtualTableAlias", value); - _virtualTableAliasField = value; - } - return this; - } - - @Override - public LateralView clone() - throws CloneNotSupportedException - { - LateralView __clone = ((LateralView) super.clone()); - __clone.__changeListener = new LateralView.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public LateralView copy() - throws CloneNotSupportedException - { - LateralView __copy = ((LateralView) super.copy()); - __copy._virtualTableAliasField = null; - __copy._tableGeneratingFunctionField = null; - __copy.__changeListener = new LateralView.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final LateralView __objectRef; - - private ChangeListener(LateralView reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "virtualTableAlias": - __objectRef._virtualTableAliasField = null; - break; - case "tableGeneratingFunction": - __objectRef._tableGeneratingFunctionField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * A table-generating function transforms a single input row to multiple output rows. For example, explode(array('A','B','C') will produce 3 one-column rows, which are row1: 'A'; row2: 'B'; row3: 'C'. - * - */ - public com.linkedin.feathr.compute.LateralView.TableGeneratingFunction.Fields tableGeneratingFunction() { - return new com.linkedin.feathr.compute.LateralView.TableGeneratingFunction.Fields(getPathComponents(), "tableGeneratingFunction"); - } - - /** - * Represents the alias for referencing the generated virtual table. It will be used in subsequent statements (eg. filter, groupBy) in the sliding window feature definition. - * - */ - public PathSpec virtualTableAlias() { - return new PathSpec(getPathComponents(), "virtualTableAlias"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.LateralView.TableGeneratingFunction.ProjectionMask _tableGeneratingFunctionMask; - - ProjectionMask() { - super(3); - } - - /** - * A table-generating function transforms a single input row to multiple output rows. For example, explode(array('A','B','C') will produce 3 one-column rows, which are row1: 'A'; row2: 'B'; row3: 'C'. - * - */ - public LateralView.ProjectionMask withTableGeneratingFunction(Function nestedMask) { - _tableGeneratingFunctionMask = nestedMask.apply(((_tableGeneratingFunctionMask == null)?LateralView.TableGeneratingFunction.createMask():_tableGeneratingFunctionMask)); - getDataMap().put("tableGeneratingFunction", _tableGeneratingFunctionMask.getDataMap()); - return this; - } - - /** - * A table-generating function transforms a single input row to multiple output rows. For example, explode(array('A','B','C') will produce 3 one-column rows, which are row1: 'A'; row2: 'B'; row3: 'C'. - * - */ - public LateralView.ProjectionMask withTableGeneratingFunction() { - _tableGeneratingFunctionMask = null; - getDataMap().put("tableGeneratingFunction", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Represents the alias for referencing the generated virtual table. It will be used in subsequent statements (eg. filter, groupBy) in the sliding window feature definition. - * - */ - public LateralView.ProjectionMask withVirtualTableAlias() { - getDataMap().put("virtualTableAlias", MaskMap.POSITIVE_MASK); - return this; - } - - } - - @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\LateralView.pdl.") - public static class TableGeneratingFunction - extends UnionTemplate - { - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}}]", SchemaFormatType.PDL)); - private com.linkedin.feathr.compute.SqlExpression _sqlExpressionMember = null; - private LateralView.TableGeneratingFunction.ChangeListener __changeListener = new LateralView.TableGeneratingFunction.ChangeListener(this); - private final static DataSchema MEMBER_SqlExpression = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.SqlExpression"); - - public TableGeneratingFunction() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public TableGeneratingFunction(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static LateralView.TableGeneratingFunction create(com.linkedin.feathr.compute.SqlExpression value) { - LateralView.TableGeneratingFunction newUnion = new LateralView.TableGeneratingFunction(); - newUnion.setSqlExpression(value); - return newUnion; - } - - public boolean isSqlExpression() { - return memberIs("com.linkedin.feathr.compute.SqlExpression"); - } - - public com.linkedin.feathr.compute.SqlExpression getSqlExpression() { - checkNotNull(); - if (_sqlExpressionMember!= null) { - return _sqlExpressionMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.SqlExpression"); - _sqlExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.SqlExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _sqlExpressionMember; - } - - public void setSqlExpression(com.linkedin.feathr.compute.SqlExpression value) { - checkNotNull(); - super._map.clear(); - _sqlExpressionMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.SqlExpression", value.data()); - } - - public static LateralView.TableGeneratingFunction.ProjectionMask createMask() { - return new LateralView.TableGeneratingFunction.ProjectionMask(); - } - - @Override - public LateralView.TableGeneratingFunction clone() - throws CloneNotSupportedException - { - LateralView.TableGeneratingFunction __clone = ((LateralView.TableGeneratingFunction) super.clone()); - __clone.__changeListener = new LateralView.TableGeneratingFunction.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public LateralView.TableGeneratingFunction copy() - throws CloneNotSupportedException - { - LateralView.TableGeneratingFunction __copy = ((LateralView.TableGeneratingFunction) super.copy()); - __copy._sqlExpressionMember = null; - __copy.__changeListener = new LateralView.TableGeneratingFunction.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final LateralView.TableGeneratingFunction __objectRef; - - private ChangeListener(LateralView.TableGeneratingFunction reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "com.linkedin.feathr.compute.SqlExpression": - __objectRef._sqlExpressionMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.compute.SqlExpression.Fields SqlExpression() { - return new com.linkedin.feathr.compute.SqlExpression.Fields(getPathComponents(), "com.linkedin.feathr.compute.SqlExpression"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.SqlExpression.ProjectionMask _SqlExpressionMask; - - ProjectionMask() { - super(2); - } - - public LateralView.TableGeneratingFunction.ProjectionMask withSqlExpression(Function nestedMask) { - _SqlExpressionMask = nestedMask.apply(((_SqlExpressionMask == null)?com.linkedin.feathr.compute.SqlExpression.createMask():_SqlExpressionMask)); - getDataMap().put("com.linkedin.feathr.compute.SqlExpression", _SqlExpressionMask.getDataMap()); - return this; - } - - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralViewArray.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralViewArray.java deleted file mode 100644 index 8a4260c3c..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/LateralViewArray.java +++ /dev/null @@ -1,118 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.schema.ArrayDataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.TemplateOutputCastException; -import com.linkedin.data.template.WrappingArrayTemplate; - -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SlidingWindowFeature.pdl.") -public class LateralViewArray - extends WrappingArrayTemplate -{ - - private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[{namespace com.linkedin.feathr.compute/**Lateral view is used in conjunction with table generating functions (eg. the most commonly used explode()), which typically generates zero or more output rows for each input row. A lateral view first applies the table generating function to each row of base table, and then joins resulting output rows to the input rows to form a virtual table with the supplied table alias. For more details and examples, refer to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+LateralView.*/record LateralView{/**A table-generating function transforms a single input row to multiple output rows. For example, explode(array('A','B','C') will produce 3 one-column rows, which are row1: 'A'; row2: 'B'; row3: 'C'.*/tableGeneratingFunction:union[/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}]/**Represents the alias for referencing the generated virtual table. It will be used in subsequent statements (eg. filter, groupBy) in the sliding window feature definition.*/virtualTableAlias:string}}]", SchemaFormatType.PDL)); - - public LateralViewArray() { - this(new DataList()); - } - - public LateralViewArray(int initialCapacity) { - this(new DataList(initialCapacity)); - } - - public LateralViewArray(Collection c) { - this(new DataList(c.size())); - addAll(c); - } - - public LateralViewArray(DataList data) { - super(data, SCHEMA, LateralView.class); - } - - public LateralViewArray(LateralView first, LateralView... rest) { - this(new DataList((rest.length + 1))); - add(first); - addAll(Arrays.asList(rest)); - } - - public static ArrayDataSchema dataSchema() { - return SCHEMA; - } - - public static LateralViewArray.ProjectionMask createMask() { - return new LateralViewArray.ProjectionMask(); - } - - @Override - public LateralViewArray clone() - throws CloneNotSupportedException - { - LateralViewArray __clone = ((LateralViewArray) super.clone()); - return __clone; - } - - @Override - public LateralViewArray copy() - throws CloneNotSupportedException - { - LateralViewArray __copy = ((LateralViewArray) super.copy()); - return __copy; - } - - @Override - protected LateralView coerceOutput(Object object) - throws TemplateOutputCastException - { - assert(object != null); - return ((object == null)?null:new LateralView(DataTemplateUtil.castOrThrow(object, DataMap.class))); - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.compute.LateralView.Fields items() { - return new com.linkedin.feathr.compute.LateralView.Fields(getPathComponents(), PathSpec.WILDCARD); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.LateralView.ProjectionMask _itemsMask; - - ProjectionMask() { - super(4); - } - - public LateralViewArray.ProjectionMask withItems(Function nestedMask) { - _itemsMask = nestedMask.apply(((_itemsMask == null)?LateralView.createMask():_itemsMask)); - getDataMap().put("$*", _itemsMask.getDataMap()); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Lookup.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Lookup.java deleted file mode 100644 index 85dba2932..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Lookup.java +++ /dev/null @@ -1,1587 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.ArrayDataSchema; -import com.linkedin.data.schema.DataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.UnionDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; -import com.linkedin.data.template.TemplateOutputCastException; -import com.linkedin.data.template.UnionTemplate; -import com.linkedin.data.template.WrappingArrayTemplate; - - -/** - * A node to represent a feature which is to be computed by using an already computed feature as the key. - * https://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?spaceKey=ENGS&title=feathr+Offline+User+Guide#FrameOfflineUserGuide-sequentialjoin - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Lookup.pdl.") -public class Lookup - extends RecordTemplate -{ - - private final static Lookup.Fields _fields = new Lookup.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**A node to represent a feature which is to be computed by using an already computed feature as the key.\r\nhttps://iwww.corp.linkedin.com/wiki/cf/pages/viewpage.action?spaceKey=ENGS&title=feathr+Offline+User+Guide#FrameOfflineUserGuide-sequentialjoin*/record Lookup includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**An array of references to a node and keys.\r\n\r\nFor now, we do not support lookup of just a key reference, but we have added that as a placeholder.\r\n\r\nA node reference consists of node id and a key reference.\r\nIn sequential join the lookup key would be a combination of the\r\nfeature node representing the base feature (lookup node) and the key associated with it. For example,:-\r\nseqJoinFeature: {\r\n base: {key: x, feature: baseFeature}\r\n expansion: {key: y, feature: expansionFeature}\r\n aggregation: UNION\r\n}\r\nHere, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would\r\npoint to the index of \"x\" in the key array of baseFeature.*/lookupKey:array[union[/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}KeyReference]]/**The node id of the node containing the expansion feature.*/lookupNode:NodeId/**Aggregation type as listed in\r\nhttps://jarvis.corp.linkedin.com/codesearch/result/\r\n?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7\r\n*/aggregation:string/**feature name of the feature which would be computed.\r\nwe need feature name here for 2 main reasons.\r\n1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and\r\nwe want to leverage that.\r\n2. For default values. Similar to above, there are existing APIs which create default value map from feature name ->\r\ndefault value.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}", SchemaFormatType.PDL)); - private Integer _idField = null; - private ConcreteKey _concreteKeyField = null; - private Lookup.LookupKeyArray _lookupKeyField = null; - private Integer _lookupNodeField = null; - private String _aggregationField = null; - private String _featureNameField = null; - private FeatureVersion _featureVersionField = null; - private Lookup.ChangeListener __changeListener = new Lookup.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); - private final static RecordDataSchema.Field FIELD_ConcreteKey = SCHEMA.getField("concreteKey"); - private final static RecordDataSchema.Field FIELD_LookupKey = SCHEMA.getField("lookupKey"); - private final static RecordDataSchema.Field FIELD_LookupNode = SCHEMA.getField("lookupNode"); - private final static RecordDataSchema.Field FIELD_Aggregation = SCHEMA.getField("aggregation"); - private final static RecordDataSchema.Field FIELD_FeatureName = SCHEMA.getField("featureName"); - private final static RecordDataSchema.Field FIELD_FeatureVersion = SCHEMA.getField("featureVersion"); - - public Lookup() { - super(new DataMap(10, 0.75F), SCHEMA, 4); - addChangeListener(__changeListener); - } - - public Lookup(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static Lookup.Fields fields() { - return _fields; - } - - public static Lookup.ProjectionMask createMask() { - return new Lookup.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for id - * - * @see Lookup.Fields#id - */ - public boolean hasId() { - if (_idField!= null) { - return true; - } - return super._map.containsKey("id"); - } - - /** - * Remover for id - * - * @see Lookup.Fields#id - */ - public void removeId() { - super._map.remove("id"); - } - - /** - * Getter for id - * - * @see Lookup.Fields#id - */ - public Integer getId(GetMode mode) { - switch (mode) { - case STRICT: - return getId(); - case DEFAULT: - case NULL: - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for id - * - * @return - * Required field. Could be null for partial record. - * @see Lookup.Fields#id - */ - @Nonnull - public Integer getId() { - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("id"); - } - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - - /** - * Setter for id - * - * @see Lookup.Fields#id - */ - public Lookup setId(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setId(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.Lookup"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeId(); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - } - return this; - } - - /** - * Setter for id - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Lookup.Fields#id - */ - public Lookup setId( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.Lookup to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - return this; - } - - /** - * Setter for id - * - * @see Lookup.Fields#id - */ - public Lookup setId(int value) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - return this; - } - - /** - * Existence checker for concreteKey - * - * @see Lookup.Fields#concreteKey - */ - public boolean hasConcreteKey() { - if (_concreteKeyField!= null) { - return true; - } - return super._map.containsKey("concreteKey"); - } - - /** - * Remover for concreteKey - * - * @see Lookup.Fields#concreteKey - */ - public void removeConcreteKey() { - super._map.remove("concreteKey"); - } - - /** - * Getter for concreteKey - * - * @see Lookup.Fields#concreteKey - */ - public ConcreteKey getConcreteKey(GetMode mode) { - return getConcreteKey(); - } - - /** - * Getter for concreteKey - * - * @return - * Optional field. Always check for null. - * @see Lookup.Fields#concreteKey - */ - @Nullable - public ConcreteKey getConcreteKey() { - if (_concreteKeyField!= null) { - return _concreteKeyField; - } else { - Object __rawValue = super._map.get("concreteKey"); - _concreteKeyField = ((__rawValue == null)?null:new ConcreteKey(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _concreteKeyField; - } - } - - /** - * Setter for concreteKey - * - * @see Lookup.Fields#concreteKey - */ - public Lookup setConcreteKey(ConcreteKey value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setConcreteKey(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeConcreteKey(); - } else { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - break; - } - return this; - } - - /** - * Setter for concreteKey - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Lookup.Fields#concreteKey - */ - public Lookup setConcreteKey( - @Nonnull - ConcreteKey value) { - if (value == null) { - throw new NullPointerException("Cannot set field concreteKey of com.linkedin.feathr.compute.Lookup to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - return this; - } - - /** - * Existence checker for lookupKey - * - * @see Lookup.Fields#lookupKey - */ - public boolean hasLookupKey() { - if (_lookupKeyField!= null) { - return true; - } - return super._map.containsKey("lookupKey"); - } - - /** - * Remover for lookupKey - * - * @see Lookup.Fields#lookupKey - */ - public void removeLookupKey() { - super._map.remove("lookupKey"); - } - - /** - * Getter for lookupKey - * - * @see Lookup.Fields#lookupKey - */ - public Lookup.LookupKeyArray getLookupKey(GetMode mode) { - switch (mode) { - case STRICT: - return getLookupKey(); - case DEFAULT: - case NULL: - if (_lookupKeyField!= null) { - return _lookupKeyField; - } else { - Object __rawValue = super._map.get("lookupKey"); - _lookupKeyField = ((__rawValue == null)?null:new Lookup.LookupKeyArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _lookupKeyField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for lookupKey - * - * @return - * Required field. Could be null for partial record. - * @see Lookup.Fields#lookupKey - */ - @Nonnull - public Lookup.LookupKeyArray getLookupKey() { - if (_lookupKeyField!= null) { - return _lookupKeyField; - } else { - Object __rawValue = super._map.get("lookupKey"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("lookupKey"); - } - _lookupKeyField = ((__rawValue == null)?null:new Lookup.LookupKeyArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _lookupKeyField; - } - } - - /** - * Setter for lookupKey - * - * @see Lookup.Fields#lookupKey - */ - public Lookup setLookupKey(Lookup.LookupKeyArray value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setLookupKey(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field lookupKey of com.linkedin.feathr.compute.Lookup"); - } else { - CheckedUtil.putWithoutChecking(super._map, "lookupKey", value.data()); - _lookupKeyField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeLookupKey(); - } else { - CheckedUtil.putWithoutChecking(super._map, "lookupKey", value.data()); - _lookupKeyField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "lookupKey", value.data()); - _lookupKeyField = value; - } - break; - } - return this; - } - - /** - * Setter for lookupKey - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Lookup.Fields#lookupKey - */ - public Lookup setLookupKey( - @Nonnull - Lookup.LookupKeyArray value) { - if (value == null) { - throw new NullPointerException("Cannot set field lookupKey of com.linkedin.feathr.compute.Lookup to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "lookupKey", value.data()); - _lookupKeyField = value; - } - return this; - } - - /** - * Existence checker for lookupNode - * - * @see Lookup.Fields#lookupNode - */ - public boolean hasLookupNode() { - if (_lookupNodeField!= null) { - return true; - } - return super._map.containsKey("lookupNode"); - } - - /** - * Remover for lookupNode - * - * @see Lookup.Fields#lookupNode - */ - public void removeLookupNode() { - super._map.remove("lookupNode"); - } - - /** - * Getter for lookupNode - * - * @see Lookup.Fields#lookupNode - */ - public Integer getLookupNode(GetMode mode) { - switch (mode) { - case STRICT: - return getLookupNode(); - case DEFAULT: - case NULL: - if (_lookupNodeField!= null) { - return _lookupNodeField; - } else { - Object __rawValue = super._map.get("lookupNode"); - _lookupNodeField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _lookupNodeField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for lookupNode - * - * @return - * Required field. Could be null for partial record. - * @see Lookup.Fields#lookupNode - */ - @Nonnull - public Integer getLookupNode() { - if (_lookupNodeField!= null) { - return _lookupNodeField; - } else { - Object __rawValue = super._map.get("lookupNode"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("lookupNode"); - } - _lookupNodeField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _lookupNodeField; - } - } - - /** - * Setter for lookupNode - * - * @see Lookup.Fields#lookupNode - */ - public Lookup setLookupNode(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setLookupNode(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field lookupNode of com.linkedin.feathr.compute.Lookup"); - } else { - CheckedUtil.putWithoutChecking(super._map, "lookupNode", DataTemplateUtil.coerceIntInput(value)); - _lookupNodeField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeLookupNode(); - } else { - CheckedUtil.putWithoutChecking(super._map, "lookupNode", DataTemplateUtil.coerceIntInput(value)); - _lookupNodeField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "lookupNode", DataTemplateUtil.coerceIntInput(value)); - _lookupNodeField = value; - } - break; - } - return this; - } - - /** - * Setter for lookupNode - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Lookup.Fields#lookupNode - */ - public Lookup setLookupNode( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field lookupNode of com.linkedin.feathr.compute.Lookup to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "lookupNode", DataTemplateUtil.coerceIntInput(value)); - _lookupNodeField = value; - } - return this; - } - - /** - * Setter for lookupNode - * - * @see Lookup.Fields#lookupNode - */ - public Lookup setLookupNode(int value) { - CheckedUtil.putWithoutChecking(super._map, "lookupNode", DataTemplateUtil.coerceIntInput(value)); - _lookupNodeField = value; - return this; - } - - /** - * Existence checker for aggregation - * - * @see Lookup.Fields#aggregation - */ - public boolean hasAggregation() { - if (_aggregationField!= null) { - return true; - } - return super._map.containsKey("aggregation"); - } - - /** - * Remover for aggregation - * - * @see Lookup.Fields#aggregation - */ - public void removeAggregation() { - super._map.remove("aggregation"); - } - - /** - * Getter for aggregation - * - * @see Lookup.Fields#aggregation - */ - public String getAggregation(GetMode mode) { - switch (mode) { - case STRICT: - return getAggregation(); - case DEFAULT: - case NULL: - if (_aggregationField!= null) { - return _aggregationField; - } else { - Object __rawValue = super._map.get("aggregation"); - _aggregationField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _aggregationField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for aggregation - * - * @return - * Required field. Could be null for partial record. - * @see Lookup.Fields#aggregation - */ - @Nonnull - public String getAggregation() { - if (_aggregationField!= null) { - return _aggregationField; - } else { - Object __rawValue = super._map.get("aggregation"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("aggregation"); - } - _aggregationField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _aggregationField; - } - } - - /** - * Setter for aggregation - * - * @see Lookup.Fields#aggregation - */ - public Lookup setAggregation(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setAggregation(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field aggregation of com.linkedin.feathr.compute.Lookup"); - } else { - CheckedUtil.putWithoutChecking(super._map, "aggregation", value); - _aggregationField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeAggregation(); - } else { - CheckedUtil.putWithoutChecking(super._map, "aggregation", value); - _aggregationField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "aggregation", value); - _aggregationField = value; - } - break; - } - return this; - } - - /** - * Setter for aggregation - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Lookup.Fields#aggregation - */ - public Lookup setAggregation( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field aggregation of com.linkedin.feathr.compute.Lookup to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "aggregation", value); - _aggregationField = value; - } - return this; - } - - /** - * Existence checker for featureName - * - * @see Lookup.Fields#featureName - */ - public boolean hasFeatureName() { - if (_featureNameField!= null) { - return true; - } - return super._map.containsKey("featureName"); - } - - /** - * Remover for featureName - * - * @see Lookup.Fields#featureName - */ - public void removeFeatureName() { - super._map.remove("featureName"); - } - - /** - * Getter for featureName - * - * @see Lookup.Fields#featureName - */ - public String getFeatureName(GetMode mode) { - switch (mode) { - case STRICT: - return getFeatureName(); - case DEFAULT: - case NULL: - if (_featureNameField!= null) { - return _featureNameField; - } else { - Object __rawValue = super._map.get("featureName"); - _featureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _featureNameField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for featureName - * - * @return - * Required field. Could be null for partial record. - * @see Lookup.Fields#featureName - */ - @Nonnull - public String getFeatureName() { - if (_featureNameField!= null) { - return _featureNameField; - } else { - Object __rawValue = super._map.get("featureName"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("featureName"); - } - _featureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _featureNameField; - } - } - - /** - * Setter for featureName - * - * @see Lookup.Fields#featureName - */ - public Lookup setFeatureName(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFeatureName(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field featureName of com.linkedin.feathr.compute.Lookup"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureName", value); - _featureNameField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeFeatureName(); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureName", value); - _featureNameField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "featureName", value); - _featureNameField = value; - } - break; - } - return this; - } - - /** - * Setter for featureName - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Lookup.Fields#featureName - */ - public Lookup setFeatureName( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field featureName of com.linkedin.feathr.compute.Lookup to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureName", value); - _featureNameField = value; - } - return this; - } - - /** - * Existence checker for featureVersion - * - * @see Lookup.Fields#featureVersion - */ - public boolean hasFeatureVersion() { - if (_featureVersionField!= null) { - return true; - } - return super._map.containsKey("featureVersion"); - } - - /** - * Remover for featureVersion - * - * @see Lookup.Fields#featureVersion - */ - public void removeFeatureVersion() { - super._map.remove("featureVersion"); - } - - /** - * Getter for featureVersion - * - * @see Lookup.Fields#featureVersion - */ - public FeatureVersion getFeatureVersion(GetMode mode) { - switch (mode) { - case STRICT: - return getFeatureVersion(); - case DEFAULT: - case NULL: - if (_featureVersionField!= null) { - return _featureVersionField; - } else { - Object __rawValue = super._map.get("featureVersion"); - _featureVersionField = ((__rawValue == null)?null:new FeatureVersion(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _featureVersionField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for featureVersion - * - * @return - * Required field. Could be null for partial record. - * @see Lookup.Fields#featureVersion - */ - @Nonnull - public FeatureVersion getFeatureVersion() { - if (_featureVersionField!= null) { - return _featureVersionField; - } else { - Object __rawValue = super._map.get("featureVersion"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("featureVersion"); - } - _featureVersionField = ((__rawValue == null)?null:new FeatureVersion(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _featureVersionField; - } - } - - /** - * Setter for featureVersion - * - * @see Lookup.Fields#featureVersion - */ - public Lookup setFeatureVersion(FeatureVersion value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFeatureVersion(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field featureVersion of com.linkedin.feathr.compute.Lookup"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); - _featureVersionField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeFeatureVersion(); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); - _featureVersionField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); - _featureVersionField = value; - } - break; - } - return this; - } - - /** - * Setter for featureVersion - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Lookup.Fields#featureVersion - */ - public Lookup setFeatureVersion( - @Nonnull - FeatureVersion value) { - if (value == null) { - throw new NullPointerException("Cannot set field featureVersion of com.linkedin.feathr.compute.Lookup to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); - _featureVersionField = value; - } - return this; - } - - @Override - public Lookup clone() - throws CloneNotSupportedException - { - Lookup __clone = ((Lookup) super.clone()); - __clone.__changeListener = new Lookup.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public Lookup copy() - throws CloneNotSupportedException - { - Lookup __copy = ((Lookup) super.copy()); - __copy._lookupKeyField = null; - __copy._featureNameField = null; - __copy._lookupNodeField = null; - __copy._aggregationField = null; - __copy._idField = null; - __copy._concreteKeyField = null; - __copy._featureVersionField = null; - __copy.__changeListener = new Lookup.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final Lookup __objectRef; - - private ChangeListener(Lookup reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "lookupKey": - __objectRef._lookupKeyField = null; - break; - case "featureName": - __objectRef._featureNameField = null; - break; - case "lookupNode": - __objectRef._lookupNodeField = null; - break; - case "aggregation": - __objectRef._aggregationField = null; - break; - case "id": - __objectRef._idField = null; - break; - case "concreteKey": - __objectRef._concreteKeyField = null; - break; - case "featureVersion": - __objectRef._featureVersionField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The node would be represented by this id. - * - */ - public PathSpec id() { - return new PathSpec(getPathComponents(), "id"); - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public com.linkedin.feathr.compute.ConcreteKey.Fields concreteKey() { - return new com.linkedin.feathr.compute.ConcreteKey.Fields(getPathComponents(), "concreteKey"); - } - - /** - * An array of references to a node and keys. - * - * For now, we do not support lookup of just a key reference, but we have added that as a placeholder. - * - * A node reference consists of node id and a key reference. - * In sequential join the lookup key would be a combination of the - * feature node representing the base feature (lookup node) and the key associated with it. For example,:- - * seqJoinFeature: { - * base: {key: x, feature: baseFeature} - * expansion: {key: y, feature: expansionFeature} - * aggregation: UNION - * } - * Here, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would - * point to the index of "x" in the key array of baseFeature. - * - */ - public com.linkedin.feathr.compute.Lookup.LookupKeyArray.Fields lookupKey() { - return new com.linkedin.feathr.compute.Lookup.LookupKeyArray.Fields(getPathComponents(), "lookupKey"); - } - - /** - * An array of references to a node and keys. - * - * For now, we do not support lookup of just a key reference, but we have added that as a placeholder. - * - * A node reference consists of node id and a key reference. - * In sequential join the lookup key would be a combination of the - * feature node representing the base feature (lookup node) and the key associated with it. For example,:- - * seqJoinFeature: { - * base: {key: x, feature: baseFeature} - * expansion: {key: y, feature: expansionFeature} - * aggregation: UNION - * } - * Here, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would - * point to the index of "x" in the key array of baseFeature. - * - */ - public PathSpec lookupKey(Integer start, Integer count) { - PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "lookupKey"); - if (start!= null) { - arrayPathSpec.setAttribute("start", start); - } - if (count!= null) { - arrayPathSpec.setAttribute("count", count); - } - return arrayPathSpec; - } - - /** - * The node id of the node containing the expansion feature. - * - */ - public PathSpec lookupNode() { - return new PathSpec(getPathComponents(), "lookupNode"); - } - - /** - * Aggregation type as listed in - * https://jarvis.corp.linkedin.com/codesearch/result/ - * ?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7 - * - * - */ - public PathSpec aggregation() { - return new PathSpec(getPathComponents(), "aggregation"); - } - - /** - * feature name of the feature which would be computed. - * we need feature name here for 2 main reasons. - * 1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and - * we want to leverage that. - * 2. For default values. Similar to above, there are existing APIs which create default value map from feature name -> - * default value. - * - */ - public PathSpec featureName() { - return new PathSpec(getPathComponents(), "featureName"); - } - - /** - * feature version of the feature - * - */ - public com.linkedin.feathr.compute.FeatureVersion.Fields featureVersion() { - return new com.linkedin.feathr.compute.FeatureVersion.Fields(getPathComponents(), "featureVersion"); - } - - } - - @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Lookup.pdl.") - public static class LookupKey - extends UnionTemplate - { - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}}com.linkedin.feathr.compute.KeyReference]", SchemaFormatType.PDL)); - private com.linkedin.feathr.compute.NodeReference _nodeReferenceMember = null; - private com.linkedin.feathr.compute.KeyReference _keyReferenceMember = null; - private Lookup.LookupKey.ChangeListener __changeListener = new Lookup.LookupKey.ChangeListener(this); - private final static DataSchema MEMBER_NodeReference = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.NodeReference"); - private final static DataSchema MEMBER_KeyReference = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.KeyReference"); - - public LookupKey() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public LookupKey(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static Lookup.LookupKey create(com.linkedin.feathr.compute.NodeReference value) { - Lookup.LookupKey newUnion = new Lookup.LookupKey(); - newUnion.setNodeReference(value); - return newUnion; - } - - public boolean isNodeReference() { - return memberIs("com.linkedin.feathr.compute.NodeReference"); - } - - public com.linkedin.feathr.compute.NodeReference getNodeReference() { - checkNotNull(); - if (_nodeReferenceMember!= null) { - return _nodeReferenceMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.NodeReference"); - _nodeReferenceMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.NodeReference(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _nodeReferenceMember; - } - - public void setNodeReference(com.linkedin.feathr.compute.NodeReference value) { - checkNotNull(); - super._map.clear(); - _nodeReferenceMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.NodeReference", value.data()); - } - - public static Lookup.LookupKey create(com.linkedin.feathr.compute.KeyReference value) { - Lookup.LookupKey newUnion = new Lookup.LookupKey(); - newUnion.setKeyReference(value); - return newUnion; - } - - public boolean isKeyReference() { - return memberIs("com.linkedin.feathr.compute.KeyReference"); - } - - public com.linkedin.feathr.compute.KeyReference getKeyReference() { - checkNotNull(); - if (_keyReferenceMember!= null) { - return _keyReferenceMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.KeyReference"); - _keyReferenceMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.KeyReference(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _keyReferenceMember; - } - - public void setKeyReference(com.linkedin.feathr.compute.KeyReference value) { - checkNotNull(); - super._map.clear(); - _keyReferenceMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.KeyReference", value.data()); - } - - public static Lookup.LookupKey.ProjectionMask createMask() { - return new Lookup.LookupKey.ProjectionMask(); - } - - @Override - public Lookup.LookupKey clone() - throws CloneNotSupportedException - { - Lookup.LookupKey __clone = ((Lookup.LookupKey) super.clone()); - __clone.__changeListener = new Lookup.LookupKey.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public Lookup.LookupKey copy() - throws CloneNotSupportedException - { - Lookup.LookupKey __copy = ((Lookup.LookupKey) super.copy()); - __copy._keyReferenceMember = null; - __copy._nodeReferenceMember = null; - __copy.__changeListener = new Lookup.LookupKey.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final Lookup.LookupKey __objectRef; - - private ChangeListener(Lookup.LookupKey reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "com.linkedin.feathr.compute.KeyReference": - __objectRef._keyReferenceMember = null; - break; - case "com.linkedin.feathr.compute.NodeReference": - __objectRef._nodeReferenceMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.compute.NodeReference.Fields NodeReference() { - return new com.linkedin.feathr.compute.NodeReference.Fields(getPathComponents(), "com.linkedin.feathr.compute.NodeReference"); - } - - public com.linkedin.feathr.compute.KeyReference.Fields KeyReference() { - return new com.linkedin.feathr.compute.KeyReference.Fields(getPathComponents(), "com.linkedin.feathr.compute.KeyReference"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.NodeReference.ProjectionMask _NodeReferenceMask; - private com.linkedin.feathr.compute.KeyReference.ProjectionMask _KeyReferenceMask; - - ProjectionMask() { - super(3); - } - - public Lookup.LookupKey.ProjectionMask withNodeReference(Function nestedMask) { - _NodeReferenceMask = nestedMask.apply(((_NodeReferenceMask == null)?com.linkedin.feathr.compute.NodeReference.createMask():_NodeReferenceMask)); - getDataMap().put("com.linkedin.feathr.compute.NodeReference", _NodeReferenceMask.getDataMap()); - return this; - } - - public Lookup.LookupKey.ProjectionMask withKeyReference(Function nestedMask) { - _KeyReferenceMask = nestedMask.apply(((_KeyReferenceMask == null)?com.linkedin.feathr.compute.KeyReference.createMask():_KeyReferenceMask)); - getDataMap().put("com.linkedin.feathr.compute.KeyReference", _KeyReferenceMask.getDataMap()); - return this; - } - - } - - } - - @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Lookup.pdl.") - public static class LookupKeyArray - extends WrappingArrayTemplate - { - - private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[union[{namespace com.linkedin.feathr.compute/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}}com.linkedin.feathr.compute.KeyReference]]", SchemaFormatType.PDL)); - - public LookupKeyArray() { - this(new DataList()); - } - - public LookupKeyArray(int initialCapacity) { - this(new DataList(initialCapacity)); - } - - public LookupKeyArray(Collection c) { - this(new DataList(c.size())); - addAll(c); - } - - public LookupKeyArray(DataList data) { - super(data, SCHEMA, Lookup.LookupKey.class); - } - - public LookupKeyArray(Lookup.LookupKey first, Lookup.LookupKey... rest) { - this(new DataList((rest.length + 1))); - add(first); - addAll(Arrays.asList(rest)); - } - - public static ArrayDataSchema dataSchema() { - return SCHEMA; - } - - public static Lookup.LookupKeyArray.ProjectionMask createMask() { - return new Lookup.LookupKeyArray.ProjectionMask(); - } - - @Override - public Lookup.LookupKeyArray clone() - throws CloneNotSupportedException - { - Lookup.LookupKeyArray __clone = ((Lookup.LookupKeyArray) super.clone()); - return __clone; - } - - @Override - public Lookup.LookupKeyArray copy() - throws CloneNotSupportedException - { - Lookup.LookupKeyArray __copy = ((Lookup.LookupKeyArray) super.copy()); - return __copy; - } - - @Override - protected Lookup.LookupKey coerceOutput(Object object) - throws TemplateOutputCastException - { - assert(object != null); - return ((object == null)?null:new Lookup.LookupKey(object)); - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.compute.Lookup.LookupKey.Fields items() { - return new com.linkedin.feathr.compute.Lookup.LookupKey.Fields(getPathComponents(), PathSpec.WILDCARD); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.Lookup.LookupKey.ProjectionMask _itemsMask; - - ProjectionMask() { - super(4); - } - - public Lookup.LookupKeyArray.ProjectionMask withItems(Function nestedMask) { - _itemsMask = nestedMask.apply(((_itemsMask == null)?Lookup.LookupKey.createMask():_itemsMask)); - getDataMap().put("$*", _itemsMask.getDataMap()); - return this; - } - - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.ConcreteKey.ProjectionMask _concreteKeyMask; - private com.linkedin.feathr.compute.Lookup.LookupKeyArray.ProjectionMask _lookupKeyMask; - private com.linkedin.feathr.compute.FeatureVersion.ProjectionMask _featureVersionMask; - - ProjectionMask() { - super(10); - } - - /** - * The node would be represented by this id. - * - */ - public Lookup.ProjectionMask withId() { - getDataMap().put("id", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public Lookup.ProjectionMask withConcreteKey(Function nestedMask) { - _concreteKeyMask = nestedMask.apply(((_concreteKeyMask == null)?ConcreteKey.createMask():_concreteKeyMask)); - getDataMap().put("concreteKey", _concreteKeyMask.getDataMap()); - return this; - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public Lookup.ProjectionMask withConcreteKey() { - _concreteKeyMask = null; - getDataMap().put("concreteKey", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * An array of references to a node and keys. - * - * For now, we do not support lookup of just a key reference, but we have added that as a placeholder. - * - * A node reference consists of node id and a key reference. - * In sequential join the lookup key would be a combination of the - * feature node representing the base feature (lookup node) and the key associated with it. For example,:- - * seqJoinFeature: { - * base: {key: x, feature: baseFeature} - * expansion: {key: y, feature: expansionFeature} - * aggregation: UNION - * } - * Here, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would - * point to the index of "x" in the key array of baseFeature. - * - */ - public Lookup.ProjectionMask withLookupKey(Function nestedMask) { - _lookupKeyMask = nestedMask.apply(((_lookupKeyMask == null)?Lookup.LookupKeyArray.createMask():_lookupKeyMask)); - getDataMap().put("lookupKey", _lookupKeyMask.getDataMap()); - return this; - } - - /** - * An array of references to a node and keys. - * - * For now, we do not support lookup of just a key reference, but we have added that as a placeholder. - * - * A node reference consists of node id and a key reference. - * In sequential join the lookup key would be a combination of the - * feature node representing the base feature (lookup node) and the key associated with it. For example,:- - * seqJoinFeature: { - * base: {key: x, feature: baseFeature} - * expansion: {key: y, feature: expansionFeature} - * aggregation: UNION - * } - * Here, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would - * point to the index of "x" in the key array of baseFeature. - * - */ - public Lookup.ProjectionMask withLookupKey() { - _lookupKeyMask = null; - getDataMap().put("lookupKey", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * An array of references to a node and keys. - * - * For now, we do not support lookup of just a key reference, but we have added that as a placeholder. - * - * A node reference consists of node id and a key reference. - * In sequential join the lookup key would be a combination of the - * feature node representing the base feature (lookup node) and the key associated with it. For example,:- - * seqJoinFeature: { - * base: {key: x, feature: baseFeature} - * expansion: {key: y, feature: expansionFeature} - * aggregation: UNION - * } - * Here, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would - * point to the index of "x" in the key array of baseFeature. - * - */ - public Lookup.ProjectionMask withLookupKey(Function nestedMask, Integer start, Integer count) { - _lookupKeyMask = nestedMask.apply(((_lookupKeyMask == null)?Lookup.LookupKeyArray.createMask():_lookupKeyMask)); - getDataMap().put("lookupKey", _lookupKeyMask.getDataMap()); - if (start!= null) { - getDataMap().getDataMap("lookupKey").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("lookupKey").put("$count", count); - } - return this; - } - - /** - * An array of references to a node and keys. - * - * For now, we do not support lookup of just a key reference, but we have added that as a placeholder. - * - * A node reference consists of node id and a key reference. - * In sequential join the lookup key would be a combination of the - * feature node representing the base feature (lookup node) and the key associated with it. For example,:- - * seqJoinFeature: { - * base: {key: x, feature: baseFeature} - * expansion: {key: y, feature: expansionFeature} - * aggregation: UNION - * } - * Here, the lookupKey's node reference would point to the node which computes the base feature, and the keyReference would - * point to the index of "x" in the key array of baseFeature. - * - */ - public Lookup.ProjectionMask withLookupKey(Integer start, Integer count) { - _lookupKeyMask = null; - getDataMap().put("lookupKey", new DataMap(3)); - if (start!= null) { - getDataMap().getDataMap("lookupKey").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("lookupKey").put("$count", count); - } - return this; - } - - /** - * The node id of the node containing the expansion feature. - * - */ - public Lookup.ProjectionMask withLookupNode() { - getDataMap().put("lookupNode", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Aggregation type as listed in - * https://jarvis.corp.linkedin.com/codesearch/result/ - * ?name=FeatureAggregationType.java&path=feathr-common%2Fframe-common%2Fsrc%2Fmain%2Fjava%2Fcom%2Flinkedin%2Fframe%2Fcommon&reponame=feathr%2Fframe-common#7 - * - * - */ - public Lookup.ProjectionMask withAggregation() { - getDataMap().put("aggregation", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * feature name of the feature which would be computed. - * we need feature name here for 2 main reasons. - * 1. For type information. There are existing APIs that create a map from feature name -> type info from FR model and - * we want to leverage that. - * 2. For default values. Similar to above, there are existing APIs which create default value map from feature name -> - * default value. - * - */ - public Lookup.ProjectionMask withFeatureName() { - getDataMap().put("featureName", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * feature version of the feature - * - */ - public Lookup.ProjectionMask withFeatureVersion(Function nestedMask) { - _featureVersionMask = nestedMask.apply(((_featureVersionMask == null)?FeatureVersion.createMask():_featureVersionMask)); - getDataMap().put("featureVersion", _featureVersionMask.getDataMap()); - return this; - } - - /** - * feature version of the feature - * - */ - public Lookup.ProjectionMask withFeatureVersion() { - _featureVersionMask = null; - getDataMap().put("featureVersion", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/MvelExpression.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/MvelExpression.java deleted file mode 100644 index e26a4db0e..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/MvelExpression.java +++ /dev/null @@ -1,260 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * An expression in MVEL language. For more information please refer to go/framemvel. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\MvelExpression.pdl.") -public class MvelExpression - extends RecordTemplate -{ - - private final static MvelExpression.Fields _fields = new MvelExpression.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**An expression in MVEL language. For more information please refer to go/framemvel.*/record MvelExpression{/**The MVEL expression.*/mvel:string}", SchemaFormatType.PDL)); - private String _mvelField = null; - private MvelExpression.ChangeListener __changeListener = new MvelExpression.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Mvel = SCHEMA.getField("mvel"); - - public MvelExpression() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public MvelExpression(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static MvelExpression.Fields fields() { - return _fields; - } - - public static MvelExpression.ProjectionMask createMask() { - return new MvelExpression.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for mvel - * - * @see MvelExpression.Fields#mvel - */ - public boolean hasMvel() { - if (_mvelField!= null) { - return true; - } - return super._map.containsKey("mvel"); - } - - /** - * Remover for mvel - * - * @see MvelExpression.Fields#mvel - */ - public void removeMvel() { - super._map.remove("mvel"); - } - - /** - * Getter for mvel - * - * @see MvelExpression.Fields#mvel - */ - public String getMvel(GetMode mode) { - switch (mode) { - case STRICT: - return getMvel(); - case DEFAULT: - case NULL: - if (_mvelField!= null) { - return _mvelField; - } else { - Object __rawValue = super._map.get("mvel"); - _mvelField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _mvelField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for mvel - * - * @return - * Required field. Could be null for partial record. - * @see MvelExpression.Fields#mvel - */ - @Nonnull - public String getMvel() { - if (_mvelField!= null) { - return _mvelField; - } else { - Object __rawValue = super._map.get("mvel"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("mvel"); - } - _mvelField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _mvelField; - } - } - - /** - * Setter for mvel - * - * @see MvelExpression.Fields#mvel - */ - public MvelExpression setMvel(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setMvel(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field mvel of com.linkedin.feathr.compute.MvelExpression"); - } else { - CheckedUtil.putWithoutChecking(super._map, "mvel", value); - _mvelField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeMvel(); - } else { - CheckedUtil.putWithoutChecking(super._map, "mvel", value); - _mvelField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "mvel", value); - _mvelField = value; - } - break; - } - return this; - } - - /** - * Setter for mvel - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see MvelExpression.Fields#mvel - */ - public MvelExpression setMvel( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field mvel of com.linkedin.feathr.compute.MvelExpression to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "mvel", value); - _mvelField = value; - } - return this; - } - - @Override - public MvelExpression clone() - throws CloneNotSupportedException - { - MvelExpression __clone = ((MvelExpression) super.clone()); - __clone.__changeListener = new MvelExpression.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public MvelExpression copy() - throws CloneNotSupportedException - { - MvelExpression __copy = ((MvelExpression) super.copy()); - __copy._mvelField = null; - __copy.__changeListener = new MvelExpression.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final MvelExpression __objectRef; - - private ChangeListener(MvelExpression reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "mvel": - __objectRef._mvelField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The MVEL expression. - * - */ - public PathSpec mvel() { - return new PathSpec(getPathComponents(), "mvel"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(2); - } - - /** - * The MVEL expression. - * - */ - public MvelExpression.ProjectionMask withMvel() { - getDataMap().put("mvel", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeId.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeId.java deleted file mode 100644 index 0b78cdadb..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeId.java +++ /dev/null @@ -1,30 +0,0 @@ - -package com.linkedin.feathr.compute; - -import javax.annotation.Generated; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.TyperefDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.TyperefInfo; - - -/** - * A type ref to int node id - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\NodeId.pdl.") -public class NodeId - extends TyperefInfo -{ - - private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**A type ref to int node id*/typeref NodeId=int", SchemaFormatType.PDL)); - - public NodeId() { - super(SCHEMA); - } - - public static TyperefDataSchema dataSchema() { - return SCHEMA; - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReference.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReference.java deleted file mode 100644 index 7d0eedf6e..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReference.java +++ /dev/null @@ -1,488 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the - * original node array. - * For example, consider:- - * anchorA: { - * key: [viewerId, vieweeId] - * feature: featureA - * } - * Let us say featureA is evaluated in node 1. - * derivation: { - * key: [vieweeId, viewerId] - * args1: {key: [vieweeId, viewerId], feature: featureA} - * definition: args1*2 - * } - * Now, the node reference (to represent args1) would be: - * nodeId: 1 - * keyReference: [1,0] - // Indicates the ordering of the key indices. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\NodeReference.pdl.") -public class NodeReference - extends RecordTemplate -{ - - private final static NodeReference.Fields _fields = new NodeReference.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}", SchemaFormatType.PDL)); - private Integer _idField = null; - private KeyReferenceArray _keyReferenceField = null; - private NodeReference.ChangeListener __changeListener = new NodeReference.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); - private final static RecordDataSchema.Field FIELD_KeyReference = SCHEMA.getField("keyReference"); - - public NodeReference() { - super(new DataMap(3, 0.75F), SCHEMA, 2); - addChangeListener(__changeListener); - } - - public NodeReference(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static NodeReference.Fields fields() { - return _fields; - } - - public static NodeReference.ProjectionMask createMask() { - return new NodeReference.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for id - * - * @see NodeReference.Fields#id - */ - public boolean hasId() { - if (_idField!= null) { - return true; - } - return super._map.containsKey("id"); - } - - /** - * Remover for id - * - * @see NodeReference.Fields#id - */ - public void removeId() { - super._map.remove("id"); - } - - /** - * Getter for id - * - * @see NodeReference.Fields#id - */ - public Integer getId(GetMode mode) { - switch (mode) { - case STRICT: - return getId(); - case DEFAULT: - case NULL: - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for id - * - * @return - * Required field. Could be null for partial record. - * @see NodeReference.Fields#id - */ - @Nonnull - public Integer getId() { - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("id"); - } - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - - /** - * Setter for id - * - * @see NodeReference.Fields#id - */ - public NodeReference setId(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setId(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.NodeReference"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeId(); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - } - return this; - } - - /** - * Setter for id - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see NodeReference.Fields#id - */ - public NodeReference setId( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.NodeReference to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - return this; - } - - /** - * Setter for id - * - * @see NodeReference.Fields#id - */ - public NodeReference setId(int value) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - return this; - } - - /** - * Existence checker for keyReference - * - * @see NodeReference.Fields#keyReference - */ - public boolean hasKeyReference() { - if (_keyReferenceField!= null) { - return true; - } - return super._map.containsKey("keyReference"); - } - - /** - * Remover for keyReference - * - * @see NodeReference.Fields#keyReference - */ - public void removeKeyReference() { - super._map.remove("keyReference"); - } - - /** - * Getter for keyReference - * - * @see NodeReference.Fields#keyReference - */ - public KeyReferenceArray getKeyReference(GetMode mode) { - switch (mode) { - case STRICT: - return getKeyReference(); - case DEFAULT: - case NULL: - if (_keyReferenceField!= null) { - return _keyReferenceField; - } else { - Object __rawValue = super._map.get("keyReference"); - _keyReferenceField = ((__rawValue == null)?null:new KeyReferenceArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _keyReferenceField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for keyReference - * - * @return - * Required field. Could be null for partial record. - * @see NodeReference.Fields#keyReference - */ - @Nonnull - public KeyReferenceArray getKeyReference() { - if (_keyReferenceField!= null) { - return _keyReferenceField; - } else { - Object __rawValue = super._map.get("keyReference"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("keyReference"); - } - _keyReferenceField = ((__rawValue == null)?null:new KeyReferenceArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _keyReferenceField; - } - } - - /** - * Setter for keyReference - * - * @see NodeReference.Fields#keyReference - */ - public NodeReference setKeyReference(KeyReferenceArray value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setKeyReference(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field keyReference of com.linkedin.feathr.compute.NodeReference"); - } else { - CheckedUtil.putWithoutChecking(super._map, "keyReference", value.data()); - _keyReferenceField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeKeyReference(); - } else { - CheckedUtil.putWithoutChecking(super._map, "keyReference", value.data()); - _keyReferenceField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "keyReference", value.data()); - _keyReferenceField = value; - } - break; - } - return this; - } - - /** - * Setter for keyReference - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see NodeReference.Fields#keyReference - */ - public NodeReference setKeyReference( - @Nonnull - KeyReferenceArray value) { - if (value == null) { - throw new NullPointerException("Cannot set field keyReference of com.linkedin.feathr.compute.NodeReference to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "keyReference", value.data()); - _keyReferenceField = value; - } - return this; - } - - @Override - public NodeReference clone() - throws CloneNotSupportedException - { - NodeReference __clone = ((NodeReference) super.clone()); - __clone.__changeListener = new NodeReference.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public NodeReference copy() - throws CloneNotSupportedException - { - NodeReference __copy = ((NodeReference) super.copy()); - __copy._keyReferenceField = null; - __copy._idField = null; - __copy.__changeListener = new NodeReference.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final NodeReference __objectRef; - - private ChangeListener(NodeReference reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "keyReference": - __objectRef._keyReferenceField = null; - break; - case "id": - __objectRef._idField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * node id of the referring node. - * - */ - public PathSpec id() { - return new PathSpec(getPathComponents(), "id"); - } - - /** - * The key references in the keys of the referring node. - * - */ - public com.linkedin.feathr.compute.KeyReferenceArray.Fields keyReference() { - return new com.linkedin.feathr.compute.KeyReferenceArray.Fields(getPathComponents(), "keyReference"); - } - - /** - * The key references in the keys of the referring node. - * - */ - public PathSpec keyReference(Integer start, Integer count) { - PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "keyReference"); - if (start!= null) { - arrayPathSpec.setAttribute("start", start); - } - if (count!= null) { - arrayPathSpec.setAttribute("count", count); - } - return arrayPathSpec; - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.KeyReferenceArray.ProjectionMask _keyReferenceMask; - - ProjectionMask() { - super(3); - } - - /** - * node id of the referring node. - * - */ - public NodeReference.ProjectionMask withId() { - getDataMap().put("id", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * The key references in the keys of the referring node. - * - */ - public NodeReference.ProjectionMask withKeyReference(Function nestedMask) { - _keyReferenceMask = nestedMask.apply(((_keyReferenceMask == null)?KeyReferenceArray.createMask():_keyReferenceMask)); - getDataMap().put("keyReference", _keyReferenceMask.getDataMap()); - return this; - } - - /** - * The key references in the keys of the referring node. - * - */ - public NodeReference.ProjectionMask withKeyReference() { - _keyReferenceMask = null; - getDataMap().put("keyReference", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * The key references in the keys of the referring node. - * - */ - public NodeReference.ProjectionMask withKeyReference(Function nestedMask, Integer start, Integer count) { - _keyReferenceMask = nestedMask.apply(((_keyReferenceMask == null)?KeyReferenceArray.createMask():_keyReferenceMask)); - getDataMap().put("keyReference", _keyReferenceMask.getDataMap()); - if (start!= null) { - getDataMap().getDataMap("keyReference").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("keyReference").put("$count", count); - } - return this; - } - - /** - * The key references in the keys of the referring node. - * - */ - public NodeReference.ProjectionMask withKeyReference(Integer start, Integer count) { - _keyReferenceMask = null; - getDataMap().put("keyReference", new DataMap(3)); - if (start!= null) { - getDataMap().getDataMap("keyReference").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("keyReference").put("$count", count); - } - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReferenceArray.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReferenceArray.java deleted file mode 100644 index 94b5d095a..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/NodeReferenceArray.java +++ /dev/null @@ -1,118 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.schema.ArrayDataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.TemplateOutputCastException; -import com.linkedin.data.template.WrappingArrayTemplate; - -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Transformation.pdl.") -public class NodeReferenceArray - extends WrappingArrayTemplate -{ - - private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[{namespace com.linkedin.feathr.compute/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}}]", SchemaFormatType.PDL)); - - public NodeReferenceArray() { - this(new DataList()); - } - - public NodeReferenceArray(int initialCapacity) { - this(new DataList(initialCapacity)); - } - - public NodeReferenceArray(Collection c) { - this(new DataList(c.size())); - addAll(c); - } - - public NodeReferenceArray(DataList data) { - super(data, SCHEMA, NodeReference.class); - } - - public NodeReferenceArray(NodeReference first, NodeReference... rest) { - this(new DataList((rest.length + 1))); - add(first); - addAll(Arrays.asList(rest)); - } - - public static ArrayDataSchema dataSchema() { - return SCHEMA; - } - - public static NodeReferenceArray.ProjectionMask createMask() { - return new NodeReferenceArray.ProjectionMask(); - } - - @Override - public NodeReferenceArray clone() - throws CloneNotSupportedException - { - NodeReferenceArray __clone = ((NodeReferenceArray) super.clone()); - return __clone; - } - - @Override - public NodeReferenceArray copy() - throws CloneNotSupportedException - { - NodeReferenceArray __copy = ((NodeReferenceArray) super.copy()); - return __copy; - } - - @Override - protected NodeReference coerceOutput(Object object) - throws TemplateOutputCastException - { - assert(object != null); - return ((object == null)?null:new NodeReference(DataTemplateUtil.castOrThrow(object, DataMap.class))); - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.compute.NodeReference.Fields items() { - return new com.linkedin.feathr.compute.NodeReference.Fields(getPathComponents(), PathSpec.WILDCARD); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.NodeReference.ProjectionMask _itemsMask; - - ProjectionMask() { - super(4); - } - - public NodeReferenceArray.ProjectionMask withItems(Function nestedMask) { - _itemsMask = nestedMask.apply(((_itemsMask == null)?NodeReference.createMask():_itemsMask)); - getDataMap().put("$*", _itemsMask.getDataMap()); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OfflineKeyFunction.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OfflineKeyFunction.java deleted file mode 100644 index 9a62a82ed..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OfflineKeyFunction.java +++ /dev/null @@ -1,477 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.DataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.UnionDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.SetMode; -import com.linkedin.data.template.UnionTemplate; - - -/** - * Represents a feature's key that is extracted from each row of an offline data source and is used to join with observation data to form a training dataset. This class is expected to be included so the definitions of enclosed fields can be reused. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\OfflineKeyFunction.pdl.") -public class OfflineKeyFunction - extends RecordTemplate -{ - - private final static OfflineKeyFunction.Fields _fields = new OfflineKeyFunction.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Represents a feature's key that is extracted from each row of an offline data source and is used to join with observation data to form a training dataset. This class is expected to be included so the definitions of enclosed fields can be reused.*/record OfflineKeyFunction{/**Key function specifies how to extract the feature's key from each row of the offline data source. For example, an offline dataset has x field, a key function being defined as getIdFromUrn(x) means the feature key is a numeric member id, which can later be used to join with observation data that also has numeric member id column. A feature's key can have one key part or multiple key parts (compound key). This field should be required, keeping it optional for fulfilling backward compatiblity requirement during schema evolution.*/keyFunction:optional union[/**An expression in MVEL language. For more information please refer to go/framemvel.*/record MvelExpression{/**The MVEL expression.*/mvel:string}/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}/**User defined function that can be used in feature extraction or derivation.*/record UserDefinedFunction{/**Reference to the class that implements the user defined function.*/clazz:string/**Some UserDefinedFunction requires additional custom parameters. This field defines the custom parameters of the user defined function, represented as a map of string to json blob. The key is the parameter name, and the value is the parameter value represented as a json blob. For example, the parameters may look like: { param1 : [\"waterlooCompany_terms_hashed\", \"waterlooCompany_values\"], param2 : \"com.linkedin.quasar.encoding.SomeEncodingClass\u00e2\u20ac\ufffd } feathr will be responsible of parsing the parameters map into a CustomParameters class defined by application: public class CustomParameters { List param1; String param2; } CustomParameters will be used in the constructor of the UserDefinedFunction.*/parameters:map[string,string]={}}]}", SchemaFormatType.PDL)); - private OfflineKeyFunction.KeyFunction _keyFunctionField = null; - private OfflineKeyFunction.ChangeListener __changeListener = new OfflineKeyFunction.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_KeyFunction = SCHEMA.getField("keyFunction"); - - public OfflineKeyFunction() { - super(new DataMap(2, 0.75F), SCHEMA, 2); - addChangeListener(__changeListener); - } - - public OfflineKeyFunction(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static OfflineKeyFunction.Fields fields() { - return _fields; - } - - public static OfflineKeyFunction.ProjectionMask createMask() { - return new OfflineKeyFunction.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for keyFunction - * - * @see OfflineKeyFunction.Fields#keyFunction - */ - public boolean hasKeyFunction() { - if (_keyFunctionField!= null) { - return true; - } - return super._map.containsKey("keyFunction"); - } - - /** - * Remover for keyFunction - * - * @see OfflineKeyFunction.Fields#keyFunction - */ - public void removeKeyFunction() { - super._map.remove("keyFunction"); - } - - /** - * Getter for keyFunction - * - * @see OfflineKeyFunction.Fields#keyFunction - */ - public OfflineKeyFunction.KeyFunction getKeyFunction(GetMode mode) { - return getKeyFunction(); - } - - /** - * Getter for keyFunction - * - * @return - * Optional field. Always check for null. - * @see OfflineKeyFunction.Fields#keyFunction - */ - @Nullable - public OfflineKeyFunction.KeyFunction getKeyFunction() { - if (_keyFunctionField!= null) { - return _keyFunctionField; - } else { - Object __rawValue = super._map.get("keyFunction"); - _keyFunctionField = ((__rawValue == null)?null:new OfflineKeyFunction.KeyFunction(__rawValue)); - return _keyFunctionField; - } - } - - /** - * Setter for keyFunction - * - * @see OfflineKeyFunction.Fields#keyFunction - */ - public OfflineKeyFunction setKeyFunction(OfflineKeyFunction.KeyFunction value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setKeyFunction(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeKeyFunction(); - } else { - CheckedUtil.putWithoutChecking(super._map, "keyFunction", value.data()); - _keyFunctionField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "keyFunction", value.data()); - _keyFunctionField = value; - } - break; - } - return this; - } - - /** - * Setter for keyFunction - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see OfflineKeyFunction.Fields#keyFunction - */ - public OfflineKeyFunction setKeyFunction( - @Nonnull - OfflineKeyFunction.KeyFunction value) { - if (value == null) { - throw new NullPointerException("Cannot set field keyFunction of com.linkedin.feathr.compute.OfflineKeyFunction to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "keyFunction", value.data()); - _keyFunctionField = value; - } - return this; - } - - @Override - public OfflineKeyFunction clone() - throws CloneNotSupportedException - { - OfflineKeyFunction __clone = ((OfflineKeyFunction) super.clone()); - __clone.__changeListener = new OfflineKeyFunction.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public OfflineKeyFunction copy() - throws CloneNotSupportedException - { - OfflineKeyFunction __copy = ((OfflineKeyFunction) super.copy()); - __copy._keyFunctionField = null; - __copy.__changeListener = new OfflineKeyFunction.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final OfflineKeyFunction __objectRef; - - private ChangeListener(OfflineKeyFunction reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "keyFunction": - __objectRef._keyFunctionField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Key function specifies how to extract the feature's key from each row of the offline data source. For example, an offline dataset has x field, a key function being defined as getIdFromUrn(x) means the feature key is a numeric member id, which can later be used to join with observation data that also has numeric member id column. A feature's key can have one key part or multiple key parts (compound key). This field should be required, keeping it optional for fulfilling backward compatiblity requirement during schema evolution. - * - */ - public com.linkedin.feathr.compute.OfflineKeyFunction.KeyFunction.Fields keyFunction() { - return new com.linkedin.feathr.compute.OfflineKeyFunction.KeyFunction.Fields(getPathComponents(), "keyFunction"); - } - - } - - @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\OfflineKeyFunction.pdl.") - public static class KeyFunction - extends UnionTemplate - { - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**An expression in MVEL language. For more information please refer to go/framemvel.*/record MvelExpression{/**The MVEL expression.*/mvel:string}}{namespace com.linkedin.feathr.compute/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}}{namespace com.linkedin.feathr.compute/**User defined function that can be used in feature extraction or derivation.*/record UserDefinedFunction{/**Reference to the class that implements the user defined function.*/clazz:string/**Some UserDefinedFunction requires additional custom parameters. This field defines the custom parameters of the user defined function, represented as a map of string to json blob. The key is the parameter name, and the value is the parameter value represented as a json blob. For example, the parameters may look like: { param1 : [\"waterlooCompany_terms_hashed\", \"waterlooCompany_values\"], param2 : \"com.linkedin.quasar.encoding.SomeEncodingClass\u00e2\u20ac\ufffd } feathr will be responsible of parsing the parameters map into a CustomParameters class defined by application: public class CustomParameters { List param1; String param2; } CustomParameters will be used in the constructor of the UserDefinedFunction.*/parameters:map[string,string]={}}}]", SchemaFormatType.PDL)); - private com.linkedin.feathr.compute.MvelExpression _mvelExpressionMember = null; - private com.linkedin.feathr.compute.SqlExpression _sqlExpressionMember = null; - private com.linkedin.feathr.compute.UserDefinedFunction _userDefinedFunctionMember = null; - private OfflineKeyFunction.KeyFunction.ChangeListener __changeListener = new OfflineKeyFunction.KeyFunction.ChangeListener(this); - private final static DataSchema MEMBER_MvelExpression = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.MvelExpression"); - private final static DataSchema MEMBER_SqlExpression = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.SqlExpression"); - private final static DataSchema MEMBER_UserDefinedFunction = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.UserDefinedFunction"); - - public KeyFunction() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public KeyFunction(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static OfflineKeyFunction.KeyFunction create(com.linkedin.feathr.compute.MvelExpression value) { - OfflineKeyFunction.KeyFunction newUnion = new OfflineKeyFunction.KeyFunction(); - newUnion.setMvelExpression(value); - return newUnion; - } - - public boolean isMvelExpression() { - return memberIs("com.linkedin.feathr.compute.MvelExpression"); - } - - public com.linkedin.feathr.compute.MvelExpression getMvelExpression() { - checkNotNull(); - if (_mvelExpressionMember!= null) { - return _mvelExpressionMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.MvelExpression"); - _mvelExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.MvelExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _mvelExpressionMember; - } - - public void setMvelExpression(com.linkedin.feathr.compute.MvelExpression value) { - checkNotNull(); - super._map.clear(); - _mvelExpressionMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.MvelExpression", value.data()); - } - - public static OfflineKeyFunction.KeyFunction create(com.linkedin.feathr.compute.SqlExpression value) { - OfflineKeyFunction.KeyFunction newUnion = new OfflineKeyFunction.KeyFunction(); - newUnion.setSqlExpression(value); - return newUnion; - } - - public boolean isSqlExpression() { - return memberIs("com.linkedin.feathr.compute.SqlExpression"); - } - - public com.linkedin.feathr.compute.SqlExpression getSqlExpression() { - checkNotNull(); - if (_sqlExpressionMember!= null) { - return _sqlExpressionMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.SqlExpression"); - _sqlExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.SqlExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _sqlExpressionMember; - } - - public void setSqlExpression(com.linkedin.feathr.compute.SqlExpression value) { - checkNotNull(); - super._map.clear(); - _sqlExpressionMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.SqlExpression", value.data()); - } - - public static OfflineKeyFunction.KeyFunction create(com.linkedin.feathr.compute.UserDefinedFunction value) { - OfflineKeyFunction.KeyFunction newUnion = new OfflineKeyFunction.KeyFunction(); - newUnion.setUserDefinedFunction(value); - return newUnion; - } - - public boolean isUserDefinedFunction() { - return memberIs("com.linkedin.feathr.compute.UserDefinedFunction"); - } - - public com.linkedin.feathr.compute.UserDefinedFunction getUserDefinedFunction() { - checkNotNull(); - if (_userDefinedFunctionMember!= null) { - return _userDefinedFunctionMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.UserDefinedFunction"); - _userDefinedFunctionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.UserDefinedFunction(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _userDefinedFunctionMember; - } - - public void setUserDefinedFunction(com.linkedin.feathr.compute.UserDefinedFunction value) { - checkNotNull(); - super._map.clear(); - _userDefinedFunctionMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.UserDefinedFunction", value.data()); - } - - public static OfflineKeyFunction.KeyFunction.ProjectionMask createMask() { - return new OfflineKeyFunction.KeyFunction.ProjectionMask(); - } - - @Override - public OfflineKeyFunction.KeyFunction clone() - throws CloneNotSupportedException - { - OfflineKeyFunction.KeyFunction __clone = ((OfflineKeyFunction.KeyFunction) super.clone()); - __clone.__changeListener = new OfflineKeyFunction.KeyFunction.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public OfflineKeyFunction.KeyFunction copy() - throws CloneNotSupportedException - { - OfflineKeyFunction.KeyFunction __copy = ((OfflineKeyFunction.KeyFunction) super.copy()); - __copy._sqlExpressionMember = null; - __copy._userDefinedFunctionMember = null; - __copy._mvelExpressionMember = null; - __copy.__changeListener = new OfflineKeyFunction.KeyFunction.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final OfflineKeyFunction.KeyFunction __objectRef; - - private ChangeListener(OfflineKeyFunction.KeyFunction reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "com.linkedin.feathr.compute.SqlExpression": - __objectRef._sqlExpressionMember = null; - break; - case "com.linkedin.feathr.compute.UserDefinedFunction": - __objectRef._userDefinedFunctionMember = null; - break; - case "com.linkedin.feathr.compute.MvelExpression": - __objectRef._mvelExpressionMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.compute.MvelExpression.Fields MvelExpression() { - return new com.linkedin.feathr.compute.MvelExpression.Fields(getPathComponents(), "com.linkedin.feathr.compute.MvelExpression"); - } - - public com.linkedin.feathr.compute.SqlExpression.Fields SqlExpression() { - return new com.linkedin.feathr.compute.SqlExpression.Fields(getPathComponents(), "com.linkedin.feathr.compute.SqlExpression"); - } - - public com.linkedin.feathr.compute.UserDefinedFunction.Fields UserDefinedFunction() { - return new com.linkedin.feathr.compute.UserDefinedFunction.Fields(getPathComponents(), "com.linkedin.feathr.compute.UserDefinedFunction"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.MvelExpression.ProjectionMask _MvelExpressionMask; - private com.linkedin.feathr.compute.SqlExpression.ProjectionMask _SqlExpressionMask; - private com.linkedin.feathr.compute.UserDefinedFunction.ProjectionMask _UserDefinedFunctionMask; - - ProjectionMask() { - super(4); - } - - public OfflineKeyFunction.KeyFunction.ProjectionMask withMvelExpression(Function nestedMask) { - _MvelExpressionMask = nestedMask.apply(((_MvelExpressionMask == null)?com.linkedin.feathr.compute.MvelExpression.createMask():_MvelExpressionMask)); - getDataMap().put("com.linkedin.feathr.compute.MvelExpression", _MvelExpressionMask.getDataMap()); - return this; - } - - public OfflineKeyFunction.KeyFunction.ProjectionMask withSqlExpression(Function nestedMask) { - _SqlExpressionMask = nestedMask.apply(((_SqlExpressionMask == null)?com.linkedin.feathr.compute.SqlExpression.createMask():_SqlExpressionMask)); - getDataMap().put("com.linkedin.feathr.compute.SqlExpression", _SqlExpressionMask.getDataMap()); - return this; - } - - public OfflineKeyFunction.KeyFunction.ProjectionMask withUserDefinedFunction(Function nestedMask) { - _UserDefinedFunctionMask = nestedMask.apply(((_UserDefinedFunctionMask == null)?com.linkedin.feathr.compute.UserDefinedFunction.createMask():_UserDefinedFunctionMask)); - getDataMap().put("com.linkedin.feathr.compute.UserDefinedFunction", _UserDefinedFunctionMask.getDataMap()); - return this; - } - - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.OfflineKeyFunction.KeyFunction.ProjectionMask _keyFunctionMask; - - ProjectionMask() { - super(2); - } - - /** - * Key function specifies how to extract the feature's key from each row of the offline data source. For example, an offline dataset has x field, a key function being defined as getIdFromUrn(x) means the feature key is a numeric member id, which can later be used to join with observation data that also has numeric member id column. A feature's key can have one key part or multiple key parts (compound key). This field should be required, keeping it optional for fulfilling backward compatiblity requirement during schema evolution. - * - */ - public OfflineKeyFunction.ProjectionMask withKeyFunction(Function nestedMask) { - _keyFunctionMask = nestedMask.apply(((_keyFunctionMask == null)?OfflineKeyFunction.KeyFunction.createMask():_keyFunctionMask)); - getDataMap().put("keyFunction", _keyFunctionMask.getDataMap()); - return this; - } - - /** - * Key function specifies how to extract the feature's key from each row of the offline data source. For example, an offline dataset has x field, a key function being defined as getIdFromUrn(x) means the feature key is a numeric member id, which can later be used to join with observation data that also has numeric member id column. A feature's key can have one key part or multiple key parts (compound key). This field should be required, keeping it optional for fulfilling backward compatiblity requirement during schema evolution. - * - */ - public OfflineKeyFunction.ProjectionMask withKeyFunction() { - _keyFunctionMask = null; - getDataMap().put("keyFunction", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OperatorId.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OperatorId.java deleted file mode 100644 index 1a1364317..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/OperatorId.java +++ /dev/null @@ -1,30 +0,0 @@ - -package com.linkedin.feathr.compute; - -import javax.annotation.Generated; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.TyperefDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.TyperefInfo; - - -/** - * operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\OperatorId.pdl.") -public class OperatorId - extends TyperefInfo -{ - - private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string", SchemaFormatType.PDL)); - - public OperatorId() { - super(SCHEMA); - } - - public static TyperefDataSchema dataSchema() { - return SCHEMA; - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SlidingWindowFeature.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SlidingWindowFeature.java deleted file mode 100644 index f6ba7dc67..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SlidingWindowFeature.java +++ /dev/null @@ -1,1574 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.DataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.UnionDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; -import com.linkedin.data.template.UnionTemplate; - - -/** - * Sliding window aggregation produces feature data by aggregating a collection of data within a given time interval into an aggregate value. It ensures point-in-time correctness, when joining with label data, feathr looks back the configurable time window from each entry's timestamp and compute the aggregagate value. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SlidingWindowFeature.pdl.") -public class SlidingWindowFeature - extends RecordTemplate -{ - - private final static SlidingWindowFeature.Fields _fields = new SlidingWindowFeature.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Sliding window aggregation produces feature data by aggregating a collection of data within a given time interval into an aggregate value. It ensures point-in-time correctness, when joining with label data, feathr looks back the configurable time window from each entry's timestamp and compute the aggregagate value.*/record SlidingWindowFeature{/**The target column to perform aggregation against.*/targetColumn:union[/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}]/**Represents supported types of aggregation.*/aggregationType:enum AggregationType{/** Sum. */SUM/** Count. */COUNT/** Max. */MAX/** Min. */MIN/** Average. */AVG/** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Max pooling is done by applying a max filter to (usually) non-overlapping subregions of the initial representation. */MAX_POOLING/** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Min pooling is done by applying a min filter to (usually) non-overlapping subregions of the initial representation. */MIN_POOLING/** Pooling is a sample-based discretization process. The objective is to down-sample an input representation and reduce its dimensionality. Average pooling is done by applying a average filter to (usually) non-overlapping subregions of the initial representation. */AVG_POOLING/** Latest */LATEST}/**Represents the time window to look back from label data's timestamp.*/window:/**Represents a time window used in sliding window algorithms.*/record Window{/**Represents the duration of the window.*/size:int/**Represents a unit of time.*/unit:enum Unit{/** A day. */DAY/** An hour. */HOUR/** A minute. */MINUTE/** A second. */SECOND}}/**Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details.*/lateralViews:array[/**Lateral view is used in conjunction with table generating functions (eg. the most commonly used explode()), which typically generates zero or more output rows for each input row. A lateral view first applies the table generating function to each row of base table, and then joins resulting output rows to the input rows to form a virtual table with the supplied table alias. For more details and examples, refer to https://cwiki.apache.org/confluence/display/Hive/LanguageManual+LateralView.*/record LateralView{/**A table-generating function transforms a single input row to multiple output rows. For example, explode(array('A','B','C') will produce 3 one-column rows, which are row1: 'A'; row2: 'B'; row3: 'C'.*/tableGeneratingFunction:union[SqlExpression]/**Represents the alias for referencing the generated virtual table. It will be used in subsequent statements (eg. filter, groupBy) in the sliding window feature definition.*/virtualTableAlias:string}]=[]/**Represents the filter statement before the aggregation.*/filter:optional union[SqlExpression]/**Represents the target to be grouped by before aggregation. If groupBy is not set, the aggregation will be performed over the entire dataset.*/groupBy:optional union[SqlExpression]/**Represents the max number of groups (with aggregation results) to return.*/limit:optional int}", SchemaFormatType.PDL)); - private SlidingWindowFeature.TargetColumn _targetColumnField = null; - private AggregationType _aggregationTypeField = null; - private Window _windowField = null; - private LateralViewArray _lateralViewsField = null; - private SlidingWindowFeature.Filter _filterField = null; - private SlidingWindowFeature.GroupBy _groupByField = null; - private Integer _limitField = null; - private SlidingWindowFeature.ChangeListener __changeListener = new SlidingWindowFeature.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_TargetColumn = SCHEMA.getField("targetColumn"); - private final static RecordDataSchema.Field FIELD_AggregationType = SCHEMA.getField("aggregationType"); - private final static RecordDataSchema.Field FIELD_Window = SCHEMA.getField("window"); - private final static RecordDataSchema.Field FIELD_LateralViews = SCHEMA.getField("lateralViews"); - private final static LateralViewArray DEFAULT_LateralViews; - private final static RecordDataSchema.Field FIELD_Filter = SCHEMA.getField("filter"); - private final static RecordDataSchema.Field FIELD_GroupBy = SCHEMA.getField("groupBy"); - private final static RecordDataSchema.Field FIELD_Limit = SCHEMA.getField("limit"); - - static { - DEFAULT_LateralViews = ((FIELD_LateralViews.getDefault() == null)?null:new LateralViewArray(DataTemplateUtil.castOrThrow(FIELD_LateralViews.getDefault(), DataList.class))); - } - - public SlidingWindowFeature() { - super(new DataMap(10, 0.75F), SCHEMA, 7); - addChangeListener(__changeListener); - } - - public SlidingWindowFeature(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static SlidingWindowFeature.Fields fields() { - return _fields; - } - - public static SlidingWindowFeature.ProjectionMask createMask() { - return new SlidingWindowFeature.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for targetColumn - * - * @see SlidingWindowFeature.Fields#targetColumn - */ - public boolean hasTargetColumn() { - if (_targetColumnField!= null) { - return true; - } - return super._map.containsKey("targetColumn"); - } - - /** - * Remover for targetColumn - * - * @see SlidingWindowFeature.Fields#targetColumn - */ - public void removeTargetColumn() { - super._map.remove("targetColumn"); - } - - /** - * Getter for targetColumn - * - * @see SlidingWindowFeature.Fields#targetColumn - */ - public SlidingWindowFeature.TargetColumn getTargetColumn(GetMode mode) { - switch (mode) { - case STRICT: - return getTargetColumn(); - case DEFAULT: - case NULL: - if (_targetColumnField!= null) { - return _targetColumnField; - } else { - Object __rawValue = super._map.get("targetColumn"); - _targetColumnField = ((__rawValue == null)?null:new SlidingWindowFeature.TargetColumn(__rawValue)); - return _targetColumnField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for targetColumn - * - * @return - * Required field. Could be null for partial record. - * @see SlidingWindowFeature.Fields#targetColumn - */ - @Nonnull - public SlidingWindowFeature.TargetColumn getTargetColumn() { - if (_targetColumnField!= null) { - return _targetColumnField; - } else { - Object __rawValue = super._map.get("targetColumn"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("targetColumn"); - } - _targetColumnField = ((__rawValue == null)?null:new SlidingWindowFeature.TargetColumn(__rawValue)); - return _targetColumnField; - } - } - - /** - * Setter for targetColumn - * - * @see SlidingWindowFeature.Fields#targetColumn - */ - public SlidingWindowFeature setTargetColumn(SlidingWindowFeature.TargetColumn value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setTargetColumn(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field targetColumn of com.linkedin.feathr.compute.SlidingWindowFeature"); - } else { - CheckedUtil.putWithoutChecking(super._map, "targetColumn", value.data()); - _targetColumnField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeTargetColumn(); - } else { - CheckedUtil.putWithoutChecking(super._map, "targetColumn", value.data()); - _targetColumnField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "targetColumn", value.data()); - _targetColumnField = value; - } - break; - } - return this; - } - - /** - * Setter for targetColumn - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see SlidingWindowFeature.Fields#targetColumn - */ - public SlidingWindowFeature setTargetColumn( - @Nonnull - SlidingWindowFeature.TargetColumn value) { - if (value == null) { - throw new NullPointerException("Cannot set field targetColumn of com.linkedin.feathr.compute.SlidingWindowFeature to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "targetColumn", value.data()); - _targetColumnField = value; - } - return this; - } - - /** - * Existence checker for aggregationType - * - * @see SlidingWindowFeature.Fields#aggregationType - */ - public boolean hasAggregationType() { - if (_aggregationTypeField!= null) { - return true; - } - return super._map.containsKey("aggregationType"); - } - - /** - * Remover for aggregationType - * - * @see SlidingWindowFeature.Fields#aggregationType - */ - public void removeAggregationType() { - super._map.remove("aggregationType"); - } - - /** - * Getter for aggregationType - * - * @see SlidingWindowFeature.Fields#aggregationType - */ - public AggregationType getAggregationType(GetMode mode) { - switch (mode) { - case STRICT: - return getAggregationType(); - case DEFAULT: - case NULL: - if (_aggregationTypeField!= null) { - return _aggregationTypeField; - } else { - Object __rawValue = super._map.get("aggregationType"); - _aggregationTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, AggregationType.class, AggregationType.$UNKNOWN); - return _aggregationTypeField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for aggregationType - * - * @return - * Required field. Could be null for partial record. - * @see SlidingWindowFeature.Fields#aggregationType - */ - @Nonnull - public AggregationType getAggregationType() { - if (_aggregationTypeField!= null) { - return _aggregationTypeField; - } else { - Object __rawValue = super._map.get("aggregationType"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("aggregationType"); - } - _aggregationTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, AggregationType.class, AggregationType.$UNKNOWN); - return _aggregationTypeField; - } - } - - /** - * Setter for aggregationType - * - * @see SlidingWindowFeature.Fields#aggregationType - */ - public SlidingWindowFeature setAggregationType(AggregationType value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setAggregationType(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field aggregationType of com.linkedin.feathr.compute.SlidingWindowFeature"); - } else { - CheckedUtil.putWithoutChecking(super._map, "aggregationType", value.name()); - _aggregationTypeField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeAggregationType(); - } else { - CheckedUtil.putWithoutChecking(super._map, "aggregationType", value.name()); - _aggregationTypeField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "aggregationType", value.name()); - _aggregationTypeField = value; - } - break; - } - return this; - } - - /** - * Setter for aggregationType - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see SlidingWindowFeature.Fields#aggregationType - */ - public SlidingWindowFeature setAggregationType( - @Nonnull - AggregationType value) { - if (value == null) { - throw new NullPointerException("Cannot set field aggregationType of com.linkedin.feathr.compute.SlidingWindowFeature to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "aggregationType", value.name()); - _aggregationTypeField = value; - } - return this; - } - - /** - * Existence checker for window - * - * @see SlidingWindowFeature.Fields#window - */ - public boolean hasWindow() { - if (_windowField!= null) { - return true; - } - return super._map.containsKey("window"); - } - - /** - * Remover for window - * - * @see SlidingWindowFeature.Fields#window - */ - public void removeWindow() { - super._map.remove("window"); - } - - /** - * Getter for window - * - * @see SlidingWindowFeature.Fields#window - */ - public Window getWindow(GetMode mode) { - switch (mode) { - case STRICT: - return getWindow(); - case DEFAULT: - case NULL: - if (_windowField!= null) { - return _windowField; - } else { - Object __rawValue = super._map.get("window"); - _windowField = ((__rawValue == null)?null:new Window(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _windowField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for window - * - * @return - * Required field. Could be null for partial record. - * @see SlidingWindowFeature.Fields#window - */ - @Nonnull - public Window getWindow() { - if (_windowField!= null) { - return _windowField; - } else { - Object __rawValue = super._map.get("window"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("window"); - } - _windowField = ((__rawValue == null)?null:new Window(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _windowField; - } - } - - /** - * Setter for window - * - * @see SlidingWindowFeature.Fields#window - */ - public SlidingWindowFeature setWindow(Window value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setWindow(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field window of com.linkedin.feathr.compute.SlidingWindowFeature"); - } else { - CheckedUtil.putWithoutChecking(super._map, "window", value.data()); - _windowField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeWindow(); - } else { - CheckedUtil.putWithoutChecking(super._map, "window", value.data()); - _windowField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "window", value.data()); - _windowField = value; - } - break; - } - return this; - } - - /** - * Setter for window - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see SlidingWindowFeature.Fields#window - */ - public SlidingWindowFeature setWindow( - @Nonnull - Window value) { - if (value == null) { - throw new NullPointerException("Cannot set field window of com.linkedin.feathr.compute.SlidingWindowFeature to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "window", value.data()); - _windowField = value; - } - return this; - } - - /** - * Existence checker for lateralViews - * - * @see SlidingWindowFeature.Fields#lateralViews - */ - public boolean hasLateralViews() { - if (_lateralViewsField!= null) { - return true; - } - return super._map.containsKey("lateralViews"); - } - - /** - * Remover for lateralViews - * - * @see SlidingWindowFeature.Fields#lateralViews - */ - public void removeLateralViews() { - super._map.remove("lateralViews"); - } - - /** - * Getter for lateralViews - * - * @see SlidingWindowFeature.Fields#lateralViews - */ - public LateralViewArray getLateralViews(GetMode mode) { - switch (mode) { - case STRICT: - case DEFAULT: - return getLateralViews(); - case NULL: - if (_lateralViewsField!= null) { - return _lateralViewsField; - } else { - Object __rawValue = super._map.get("lateralViews"); - _lateralViewsField = ((__rawValue == null)?null:new LateralViewArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _lateralViewsField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for lateralViews - * - * @return - * Required field. Could be null for partial record. - * @see SlidingWindowFeature.Fields#lateralViews - */ - @Nonnull - public LateralViewArray getLateralViews() { - if (_lateralViewsField!= null) { - return _lateralViewsField; - } else { - Object __rawValue = super._map.get("lateralViews"); - if (__rawValue == null) { - return DEFAULT_LateralViews; - } - _lateralViewsField = ((__rawValue == null)?null:new LateralViewArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _lateralViewsField; - } - } - - /** - * Setter for lateralViews - * - * @see SlidingWindowFeature.Fields#lateralViews - */ - public SlidingWindowFeature setLateralViews(LateralViewArray value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setLateralViews(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field lateralViews of com.linkedin.feathr.compute.SlidingWindowFeature"); - } else { - CheckedUtil.putWithoutChecking(super._map, "lateralViews", value.data()); - _lateralViewsField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeLateralViews(); - } else { - CheckedUtil.putWithoutChecking(super._map, "lateralViews", value.data()); - _lateralViewsField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "lateralViews", value.data()); - _lateralViewsField = value; - } - break; - } - return this; - } - - /** - * Setter for lateralViews - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see SlidingWindowFeature.Fields#lateralViews - */ - public SlidingWindowFeature setLateralViews( - @Nonnull - LateralViewArray value) { - if (value == null) { - throw new NullPointerException("Cannot set field lateralViews of com.linkedin.feathr.compute.SlidingWindowFeature to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "lateralViews", value.data()); - _lateralViewsField = value; - } - return this; - } - - /** - * Existence checker for filter - * - * @see SlidingWindowFeature.Fields#filter - */ - public boolean hasFilter() { - if (_filterField!= null) { - return true; - } - return super._map.containsKey("filter"); - } - - /** - * Remover for filter - * - * @see SlidingWindowFeature.Fields#filter - */ - public void removeFilter() { - super._map.remove("filter"); - } - - /** - * Getter for filter - * - * @see SlidingWindowFeature.Fields#filter - */ - public SlidingWindowFeature.Filter getFilter(GetMode mode) { - return getFilter(); - } - - /** - * Getter for filter - * - * @return - * Optional field. Always check for null. - * @see SlidingWindowFeature.Fields#filter - */ - @Nullable - public SlidingWindowFeature.Filter getFilter() { - if (_filterField!= null) { - return _filterField; - } else { - Object __rawValue = super._map.get("filter"); - _filterField = ((__rawValue == null)?null:new SlidingWindowFeature.Filter(__rawValue)); - return _filterField; - } - } - - /** - * Setter for filter - * - * @see SlidingWindowFeature.Fields#filter - */ - public SlidingWindowFeature setFilter(SlidingWindowFeature.Filter value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFilter(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeFilter(); - } else { - CheckedUtil.putWithoutChecking(super._map, "filter", value.data()); - _filterField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "filter", value.data()); - _filterField = value; - } - break; - } - return this; - } - - /** - * Setter for filter - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see SlidingWindowFeature.Fields#filter - */ - public SlidingWindowFeature setFilter( - @Nonnull - SlidingWindowFeature.Filter value) { - if (value == null) { - throw new NullPointerException("Cannot set field filter of com.linkedin.feathr.compute.SlidingWindowFeature to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "filter", value.data()); - _filterField = value; - } - return this; - } - - /** - * Existence checker for groupBy - * - * @see SlidingWindowFeature.Fields#groupBy - */ - public boolean hasGroupBy() { - if (_groupByField!= null) { - return true; - } - return super._map.containsKey("groupBy"); - } - - /** - * Remover for groupBy - * - * @see SlidingWindowFeature.Fields#groupBy - */ - public void removeGroupBy() { - super._map.remove("groupBy"); - } - - /** - * Getter for groupBy - * - * @see SlidingWindowFeature.Fields#groupBy - */ - public SlidingWindowFeature.GroupBy getGroupBy(GetMode mode) { - return getGroupBy(); - } - - /** - * Getter for groupBy - * - * @return - * Optional field. Always check for null. - * @see SlidingWindowFeature.Fields#groupBy - */ - @Nullable - public SlidingWindowFeature.GroupBy getGroupBy() { - if (_groupByField!= null) { - return _groupByField; - } else { - Object __rawValue = super._map.get("groupBy"); - _groupByField = ((__rawValue == null)?null:new SlidingWindowFeature.GroupBy(__rawValue)); - return _groupByField; - } - } - - /** - * Setter for groupBy - * - * @see SlidingWindowFeature.Fields#groupBy - */ - public SlidingWindowFeature setGroupBy(SlidingWindowFeature.GroupBy value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setGroupBy(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeGroupBy(); - } else { - CheckedUtil.putWithoutChecking(super._map, "groupBy", value.data()); - _groupByField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "groupBy", value.data()); - _groupByField = value; - } - break; - } - return this; - } - - /** - * Setter for groupBy - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see SlidingWindowFeature.Fields#groupBy - */ - public SlidingWindowFeature setGroupBy( - @Nonnull - SlidingWindowFeature.GroupBy value) { - if (value == null) { - throw new NullPointerException("Cannot set field groupBy of com.linkedin.feathr.compute.SlidingWindowFeature to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "groupBy", value.data()); - _groupByField = value; - } - return this; - } - - /** - * Existence checker for limit - * - * @see SlidingWindowFeature.Fields#limit - */ - public boolean hasLimit() { - if (_limitField!= null) { - return true; - } - return super._map.containsKey("limit"); - } - - /** - * Remover for limit - * - * @see SlidingWindowFeature.Fields#limit - */ - public void removeLimit() { - super._map.remove("limit"); - } - - /** - * Getter for limit - * - * @see SlidingWindowFeature.Fields#limit - */ - public Integer getLimit(GetMode mode) { - return getLimit(); - } - - /** - * Getter for limit - * - * @return - * Optional field. Always check for null. - * @see SlidingWindowFeature.Fields#limit - */ - @Nullable - public Integer getLimit() { - if (_limitField!= null) { - return _limitField; - } else { - Object __rawValue = super._map.get("limit"); - _limitField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _limitField; - } - } - - /** - * Setter for limit - * - * @see SlidingWindowFeature.Fields#limit - */ - public SlidingWindowFeature setLimit(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setLimit(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeLimit(); - } else { - CheckedUtil.putWithoutChecking(super._map, "limit", DataTemplateUtil.coerceIntInput(value)); - _limitField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "limit", DataTemplateUtil.coerceIntInput(value)); - _limitField = value; - } - break; - } - return this; - } - - /** - * Setter for limit - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see SlidingWindowFeature.Fields#limit - */ - public SlidingWindowFeature setLimit( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field limit of com.linkedin.feathr.compute.SlidingWindowFeature to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "limit", DataTemplateUtil.coerceIntInput(value)); - _limitField = value; - } - return this; - } - - /** - * Setter for limit - * - * @see SlidingWindowFeature.Fields#limit - */ - public SlidingWindowFeature setLimit(int value) { - CheckedUtil.putWithoutChecking(super._map, "limit", DataTemplateUtil.coerceIntInput(value)); - _limitField = value; - return this; - } - - @Override - public SlidingWindowFeature clone() - throws CloneNotSupportedException - { - SlidingWindowFeature __clone = ((SlidingWindowFeature) super.clone()); - __clone.__changeListener = new SlidingWindowFeature.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public SlidingWindowFeature copy() - throws CloneNotSupportedException - { - SlidingWindowFeature __copy = ((SlidingWindowFeature) super.copy()); - __copy._filterField = null; - __copy._aggregationTypeField = null; - __copy._targetColumnField = null; - __copy._limitField = null; - __copy._windowField = null; - __copy._groupByField = null; - __copy._lateralViewsField = null; - __copy.__changeListener = new SlidingWindowFeature.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final SlidingWindowFeature __objectRef; - - private ChangeListener(SlidingWindowFeature reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "filter": - __objectRef._filterField = null; - break; - case "aggregationType": - __objectRef._aggregationTypeField = null; - break; - case "targetColumn": - __objectRef._targetColumnField = null; - break; - case "limit": - __objectRef._limitField = null; - break; - case "window": - __objectRef._windowField = null; - break; - case "groupBy": - __objectRef._groupByField = null; - break; - case "lateralViews": - __objectRef._lateralViewsField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The target column to perform aggregation against. - * - */ - public com.linkedin.feathr.compute.SlidingWindowFeature.TargetColumn.Fields targetColumn() { - return new com.linkedin.feathr.compute.SlidingWindowFeature.TargetColumn.Fields(getPathComponents(), "targetColumn"); - } - - /** - * Represents supported types of aggregation. - * - */ - public PathSpec aggregationType() { - return new PathSpec(getPathComponents(), "aggregationType"); - } - - /** - * Represents the time window to look back from label data's timestamp. - * - */ - public com.linkedin.feathr.compute.Window.Fields window() { - return new com.linkedin.feathr.compute.Window.Fields(getPathComponents(), "window"); - } - - /** - * Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details. - * - */ - public com.linkedin.feathr.compute.LateralViewArray.Fields lateralViews() { - return new com.linkedin.feathr.compute.LateralViewArray.Fields(getPathComponents(), "lateralViews"); - } - - /** - * Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details. - * - */ - public PathSpec lateralViews(Integer start, Integer count) { - PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "lateralViews"); - if (start!= null) { - arrayPathSpec.setAttribute("start", start); - } - if (count!= null) { - arrayPathSpec.setAttribute("count", count); - } - return arrayPathSpec; - } - - /** - * Represents the filter statement before the aggregation. - * - */ - public com.linkedin.feathr.compute.SlidingWindowFeature.Filter.Fields filter() { - return new com.linkedin.feathr.compute.SlidingWindowFeature.Filter.Fields(getPathComponents(), "filter"); - } - - /** - * Represents the target to be grouped by before aggregation. If groupBy is not set, the aggregation will be performed over the entire dataset. - * - */ - public com.linkedin.feathr.compute.SlidingWindowFeature.GroupBy.Fields groupBy() { - return new com.linkedin.feathr.compute.SlidingWindowFeature.GroupBy.Fields(getPathComponents(), "groupBy"); - } - - /** - * Represents the max number of groups (with aggregation results) to return. - * - */ - public PathSpec limit() { - return new PathSpec(getPathComponents(), "limit"); - } - - } - - @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SlidingWindowFeature.pdl.") - public static class Filter - extends UnionTemplate - { - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}}]", SchemaFormatType.PDL)); - private com.linkedin.feathr.compute.SqlExpression _sqlExpressionMember = null; - private SlidingWindowFeature.Filter.ChangeListener __changeListener = new SlidingWindowFeature.Filter.ChangeListener(this); - private final static DataSchema MEMBER_SqlExpression = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.SqlExpression"); - - public Filter() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public Filter(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static SlidingWindowFeature.Filter create(com.linkedin.feathr.compute.SqlExpression value) { - SlidingWindowFeature.Filter newUnion = new SlidingWindowFeature.Filter(); - newUnion.setSqlExpression(value); - return newUnion; - } - - public boolean isSqlExpression() { - return memberIs("com.linkedin.feathr.compute.SqlExpression"); - } - - public com.linkedin.feathr.compute.SqlExpression getSqlExpression() { - checkNotNull(); - if (_sqlExpressionMember!= null) { - return _sqlExpressionMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.SqlExpression"); - _sqlExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.SqlExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _sqlExpressionMember; - } - - public void setSqlExpression(com.linkedin.feathr.compute.SqlExpression value) { - checkNotNull(); - super._map.clear(); - _sqlExpressionMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.SqlExpression", value.data()); - } - - public static SlidingWindowFeature.Filter.ProjectionMask createMask() { - return new SlidingWindowFeature.Filter.ProjectionMask(); - } - - @Override - public SlidingWindowFeature.Filter clone() - throws CloneNotSupportedException - { - SlidingWindowFeature.Filter __clone = ((SlidingWindowFeature.Filter) super.clone()); - __clone.__changeListener = new SlidingWindowFeature.Filter.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public SlidingWindowFeature.Filter copy() - throws CloneNotSupportedException - { - SlidingWindowFeature.Filter __copy = ((SlidingWindowFeature.Filter) super.copy()); - __copy._sqlExpressionMember = null; - __copy.__changeListener = new SlidingWindowFeature.Filter.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final SlidingWindowFeature.Filter __objectRef; - - private ChangeListener(SlidingWindowFeature.Filter reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "com.linkedin.feathr.compute.SqlExpression": - __objectRef._sqlExpressionMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.compute.SqlExpression.Fields SqlExpression() { - return new com.linkedin.feathr.compute.SqlExpression.Fields(getPathComponents(), "com.linkedin.feathr.compute.SqlExpression"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.SqlExpression.ProjectionMask _SqlExpressionMask; - - ProjectionMask() { - super(2); - } - - public SlidingWindowFeature.Filter.ProjectionMask withSqlExpression(Function nestedMask) { - _SqlExpressionMask = nestedMask.apply(((_SqlExpressionMask == null)?com.linkedin.feathr.compute.SqlExpression.createMask():_SqlExpressionMask)); - getDataMap().put("com.linkedin.feathr.compute.SqlExpression", _SqlExpressionMask.getDataMap()); - return this; - } - - } - - } - - @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SlidingWindowFeature.pdl.") - public static class GroupBy - extends UnionTemplate - { - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}}]", SchemaFormatType.PDL)); - private com.linkedin.feathr.compute.SqlExpression _sqlExpressionMember = null; - private SlidingWindowFeature.GroupBy.ChangeListener __changeListener = new SlidingWindowFeature.GroupBy.ChangeListener(this); - private final static DataSchema MEMBER_SqlExpression = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.SqlExpression"); - - public GroupBy() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public GroupBy(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static SlidingWindowFeature.GroupBy create(com.linkedin.feathr.compute.SqlExpression value) { - SlidingWindowFeature.GroupBy newUnion = new SlidingWindowFeature.GroupBy(); - newUnion.setSqlExpression(value); - return newUnion; - } - - public boolean isSqlExpression() { - return memberIs("com.linkedin.feathr.compute.SqlExpression"); - } - - public com.linkedin.feathr.compute.SqlExpression getSqlExpression() { - checkNotNull(); - if (_sqlExpressionMember!= null) { - return _sqlExpressionMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.SqlExpression"); - _sqlExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.SqlExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _sqlExpressionMember; - } - - public void setSqlExpression(com.linkedin.feathr.compute.SqlExpression value) { - checkNotNull(); - super._map.clear(); - _sqlExpressionMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.SqlExpression", value.data()); - } - - public static SlidingWindowFeature.GroupBy.ProjectionMask createMask() { - return new SlidingWindowFeature.GroupBy.ProjectionMask(); - } - - @Override - public SlidingWindowFeature.GroupBy clone() - throws CloneNotSupportedException - { - SlidingWindowFeature.GroupBy __clone = ((SlidingWindowFeature.GroupBy) super.clone()); - __clone.__changeListener = new SlidingWindowFeature.GroupBy.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public SlidingWindowFeature.GroupBy copy() - throws CloneNotSupportedException - { - SlidingWindowFeature.GroupBy __copy = ((SlidingWindowFeature.GroupBy) super.copy()); - __copy._sqlExpressionMember = null; - __copy.__changeListener = new SlidingWindowFeature.GroupBy.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final SlidingWindowFeature.GroupBy __objectRef; - - private ChangeListener(SlidingWindowFeature.GroupBy reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "com.linkedin.feathr.compute.SqlExpression": - __objectRef._sqlExpressionMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.compute.SqlExpression.Fields SqlExpression() { - return new com.linkedin.feathr.compute.SqlExpression.Fields(getPathComponents(), "com.linkedin.feathr.compute.SqlExpression"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.SqlExpression.ProjectionMask _SqlExpressionMask; - - ProjectionMask() { - super(2); - } - - public SlidingWindowFeature.GroupBy.ProjectionMask withSqlExpression(Function nestedMask) { - _SqlExpressionMask = nestedMask.apply(((_SqlExpressionMask == null)?com.linkedin.feathr.compute.SqlExpression.createMask():_SqlExpressionMask)); - getDataMap().put("com.linkedin.feathr.compute.SqlExpression", _SqlExpressionMask.getDataMap()); - return this; - } - - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.SlidingWindowFeature.TargetColumn.ProjectionMask _targetColumnMask; - private com.linkedin.feathr.compute.Window.ProjectionMask _windowMask; - private com.linkedin.feathr.compute.LateralViewArray.ProjectionMask _lateralViewsMask; - private com.linkedin.feathr.compute.SlidingWindowFeature.Filter.ProjectionMask _filterMask; - private com.linkedin.feathr.compute.SlidingWindowFeature.GroupBy.ProjectionMask _groupByMask; - - ProjectionMask() { - super(10); - } - - /** - * The target column to perform aggregation against. - * - */ - public SlidingWindowFeature.ProjectionMask withTargetColumn(Function nestedMask) { - _targetColumnMask = nestedMask.apply(((_targetColumnMask == null)?SlidingWindowFeature.TargetColumn.createMask():_targetColumnMask)); - getDataMap().put("targetColumn", _targetColumnMask.getDataMap()); - return this; - } - - /** - * The target column to perform aggregation against. - * - */ - public SlidingWindowFeature.ProjectionMask withTargetColumn() { - _targetColumnMask = null; - getDataMap().put("targetColumn", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Represents supported types of aggregation. - * - */ - public SlidingWindowFeature.ProjectionMask withAggregationType() { - getDataMap().put("aggregationType", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Represents the time window to look back from label data's timestamp. - * - */ - public SlidingWindowFeature.ProjectionMask withWindow(Function nestedMask) { - _windowMask = nestedMask.apply(((_windowMask == null)?Window.createMask():_windowMask)); - getDataMap().put("window", _windowMask.getDataMap()); - return this; - } - - /** - * Represents the time window to look back from label data's timestamp. - * - */ - public SlidingWindowFeature.ProjectionMask withWindow() { - _windowMask = null; - getDataMap().put("window", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details. - * - */ - public SlidingWindowFeature.ProjectionMask withLateralViews(Function nestedMask) { - _lateralViewsMask = nestedMask.apply(((_lateralViewsMask == null)?LateralViewArray.createMask():_lateralViewsMask)); - getDataMap().put("lateralViews", _lateralViewsMask.getDataMap()); - return this; - } - - /** - * Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details. - * - */ - public SlidingWindowFeature.ProjectionMask withLateralViews() { - _lateralViewsMask = null; - getDataMap().put("lateralViews", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details. - * - */ - public SlidingWindowFeature.ProjectionMask withLateralViews(Function nestedMask, Integer start, Integer count) { - _lateralViewsMask = nestedMask.apply(((_lateralViewsMask == null)?LateralViewArray.createMask():_lateralViewsMask)); - getDataMap().put("lateralViews", _lateralViewsMask.getDataMap()); - if (start!= null) { - getDataMap().getDataMap("lateralViews").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("lateralViews").put("$count", count); - } - return this; - } - - /** - * Represents lateral view statements to be applied before the aggregation. Refer to LateralView for more details. - * - */ - public SlidingWindowFeature.ProjectionMask withLateralViews(Integer start, Integer count) { - _lateralViewsMask = null; - getDataMap().put("lateralViews", new DataMap(3)); - if (start!= null) { - getDataMap().getDataMap("lateralViews").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("lateralViews").put("$count", count); - } - return this; - } - - /** - * Represents the filter statement before the aggregation. - * - */ - public SlidingWindowFeature.ProjectionMask withFilter(Function nestedMask) { - _filterMask = nestedMask.apply(((_filterMask == null)?SlidingWindowFeature.Filter.createMask():_filterMask)); - getDataMap().put("filter", _filterMask.getDataMap()); - return this; - } - - /** - * Represents the filter statement before the aggregation. - * - */ - public SlidingWindowFeature.ProjectionMask withFilter() { - _filterMask = null; - getDataMap().put("filter", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Represents the target to be grouped by before aggregation. If groupBy is not set, the aggregation will be performed over the entire dataset. - * - */ - public SlidingWindowFeature.ProjectionMask withGroupBy(Function nestedMask) { - _groupByMask = nestedMask.apply(((_groupByMask == null)?SlidingWindowFeature.GroupBy.createMask():_groupByMask)); - getDataMap().put("groupBy", _groupByMask.getDataMap()); - return this; - } - - /** - * Represents the target to be grouped by before aggregation. If groupBy is not set, the aggregation will be performed over the entire dataset. - * - */ - public SlidingWindowFeature.ProjectionMask withGroupBy() { - _groupByMask = null; - getDataMap().put("groupBy", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Represents the max number of groups (with aggregation results) to return. - * - */ - public SlidingWindowFeature.ProjectionMask withLimit() { - getDataMap().put("limit", MaskMap.POSITIVE_MASK); - return this; - } - - } - - @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SlidingWindowFeature.pdl.") - public static class TargetColumn - extends UnionTemplate - { - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[{namespace com.linkedin.feathr.compute/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}}]", SchemaFormatType.PDL)); - private com.linkedin.feathr.compute.SqlExpression _sqlExpressionMember = null; - private SlidingWindowFeature.TargetColumn.ChangeListener __changeListener = new SlidingWindowFeature.TargetColumn.ChangeListener(this); - private final static DataSchema MEMBER_SqlExpression = SCHEMA.getTypeByMemberKey("com.linkedin.feathr.compute.SqlExpression"); - - public TargetColumn() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public TargetColumn(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static SlidingWindowFeature.TargetColumn create(com.linkedin.feathr.compute.SqlExpression value) { - SlidingWindowFeature.TargetColumn newUnion = new SlidingWindowFeature.TargetColumn(); - newUnion.setSqlExpression(value); - return newUnion; - } - - public boolean isSqlExpression() { - return memberIs("com.linkedin.feathr.compute.SqlExpression"); - } - - public com.linkedin.feathr.compute.SqlExpression getSqlExpression() { - checkNotNull(); - if (_sqlExpressionMember!= null) { - return _sqlExpressionMember; - } - Object __rawValue = super._map.get("com.linkedin.feathr.compute.SqlExpression"); - _sqlExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.compute.SqlExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _sqlExpressionMember; - } - - public void setSqlExpression(com.linkedin.feathr.compute.SqlExpression value) { - checkNotNull(); - super._map.clear(); - _sqlExpressionMember = value; - CheckedUtil.putWithoutChecking(super._map, "com.linkedin.feathr.compute.SqlExpression", value.data()); - } - - public static SlidingWindowFeature.TargetColumn.ProjectionMask createMask() { - return new SlidingWindowFeature.TargetColumn.ProjectionMask(); - } - - @Override - public SlidingWindowFeature.TargetColumn clone() - throws CloneNotSupportedException - { - SlidingWindowFeature.TargetColumn __clone = ((SlidingWindowFeature.TargetColumn) super.clone()); - __clone.__changeListener = new SlidingWindowFeature.TargetColumn.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public SlidingWindowFeature.TargetColumn copy() - throws CloneNotSupportedException - { - SlidingWindowFeature.TargetColumn __copy = ((SlidingWindowFeature.TargetColumn) super.copy()); - __copy._sqlExpressionMember = null; - __copy.__changeListener = new SlidingWindowFeature.TargetColumn.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final SlidingWindowFeature.TargetColumn __objectRef; - - private ChangeListener(SlidingWindowFeature.TargetColumn reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "com.linkedin.feathr.compute.SqlExpression": - __objectRef._sqlExpressionMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.compute.SqlExpression.Fields SqlExpression() { - return new com.linkedin.feathr.compute.SqlExpression.Fields(getPathComponents(), "com.linkedin.feathr.compute.SqlExpression"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.SqlExpression.ProjectionMask _SqlExpressionMask; - - ProjectionMask() { - super(2); - } - - public SlidingWindowFeature.TargetColumn.ProjectionMask withSqlExpression(Function nestedMask) { - _SqlExpressionMask = nestedMask.apply(((_SqlExpressionMask == null)?com.linkedin.feathr.compute.SqlExpression.createMask():_SqlExpressionMask)); - getDataMap().put("com.linkedin.feathr.compute.SqlExpression", _SqlExpressionMask.getDataMap()); - return this; - } - - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SqlExpression.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SqlExpression.java deleted file mode 100644 index 53e075683..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/SqlExpression.java +++ /dev/null @@ -1,260 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * An expression in Spark SQL. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\SqlExpression.pdl.") -public class SqlExpression - extends RecordTemplate -{ - - private final static SqlExpression.Fields _fields = new SqlExpression.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**An expression in Spark SQL.*/record SqlExpression{/**The Spark SQL expression.*/sql:string}", SchemaFormatType.PDL)); - private String _sqlField = null; - private SqlExpression.ChangeListener __changeListener = new SqlExpression.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Sql = SCHEMA.getField("sql"); - - public SqlExpression() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public SqlExpression(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static SqlExpression.Fields fields() { - return _fields; - } - - public static SqlExpression.ProjectionMask createMask() { - return new SqlExpression.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for sql - * - * @see SqlExpression.Fields#sql - */ - public boolean hasSql() { - if (_sqlField!= null) { - return true; - } - return super._map.containsKey("sql"); - } - - /** - * Remover for sql - * - * @see SqlExpression.Fields#sql - */ - public void removeSql() { - super._map.remove("sql"); - } - - /** - * Getter for sql - * - * @see SqlExpression.Fields#sql - */ - public String getSql(GetMode mode) { - switch (mode) { - case STRICT: - return getSql(); - case DEFAULT: - case NULL: - if (_sqlField!= null) { - return _sqlField; - } else { - Object __rawValue = super._map.get("sql"); - _sqlField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _sqlField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for sql - * - * @return - * Required field. Could be null for partial record. - * @see SqlExpression.Fields#sql - */ - @Nonnull - public String getSql() { - if (_sqlField!= null) { - return _sqlField; - } else { - Object __rawValue = super._map.get("sql"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("sql"); - } - _sqlField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _sqlField; - } - } - - /** - * Setter for sql - * - * @see SqlExpression.Fields#sql - */ - public SqlExpression setSql(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setSql(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field sql of com.linkedin.feathr.compute.SqlExpression"); - } else { - CheckedUtil.putWithoutChecking(super._map, "sql", value); - _sqlField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeSql(); - } else { - CheckedUtil.putWithoutChecking(super._map, "sql", value); - _sqlField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "sql", value); - _sqlField = value; - } - break; - } - return this; - } - - /** - * Setter for sql - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see SqlExpression.Fields#sql - */ - public SqlExpression setSql( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field sql of com.linkedin.feathr.compute.SqlExpression to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "sql", value); - _sqlField = value; - } - return this; - } - - @Override - public SqlExpression clone() - throws CloneNotSupportedException - { - SqlExpression __clone = ((SqlExpression) super.clone()); - __clone.__changeListener = new SqlExpression.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public SqlExpression copy() - throws CloneNotSupportedException - { - SqlExpression __copy = ((SqlExpression) super.copy()); - __copy._sqlField = null; - __copy.__changeListener = new SqlExpression.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final SqlExpression __objectRef; - - private ChangeListener(SqlExpression reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "sql": - __objectRef._sqlField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The Spark SQL expression. - * - */ - public PathSpec sql() { - return new PathSpec(getPathComponents(), "sql"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(2); - } - - /** - * The Spark SQL expression. - * - */ - public SqlExpression.ProjectionMask withSql() { - getDataMap().put("sql", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorCategory.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorCategory.java deleted file mode 100644 index 6fc37fcc4..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorCategory.java +++ /dev/null @@ -1,42 +0,0 @@ - -package com.linkedin.feathr.compute; - -import javax.annotation.Generated; -import com.linkedin.data.schema.EnumDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; - - -/** - * Supported Tensor categories in feathr and Quince. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\TensorCategory.pdl.") -public enum TensorCategory { - - - /** - * Dense tensors store values in a contiguous sequential block of memory where all values are represented. - * - */ - DENSE, - - /** - * Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them. - * - */ - SPARSE, - - /** - * Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions. - * - */ - RAGGED, - $UNKNOWN; - private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}", SchemaFormatType.PDL)); - - public static EnumDataSchema dataSchema() { - return SCHEMA; - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorFeatureFormat.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorFeatureFormat.java deleted file mode 100644 index a869fdbbc..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TensorFeatureFormat.java +++ /dev/null @@ -1,603 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\TensorFeatureFormat.pdl.") -public class TensorFeatureFormat - extends RecordTemplate -{ - - private final static TensorFeatureFormat.Fields _fields = new TensorFeatureFormat.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}", SchemaFormatType.PDL)); - private TensorCategory _tensorCategoryField = null; - private ValueType _valueTypeField = null; - private DimensionArray _dimensionsField = null; - private TensorFeatureFormat.ChangeListener __changeListener = new TensorFeatureFormat.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_TensorCategory = SCHEMA.getField("tensorCategory"); - private final static RecordDataSchema.Field FIELD_ValueType = SCHEMA.getField("valueType"); - private final static RecordDataSchema.Field FIELD_Dimensions = SCHEMA.getField("dimensions"); - - public TensorFeatureFormat() { - super(new DataMap(4, 0.75F), SCHEMA, 2); - addChangeListener(__changeListener); - } - - public TensorFeatureFormat(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static TensorFeatureFormat.Fields fields() { - return _fields; - } - - public static TensorFeatureFormat.ProjectionMask createMask() { - return new TensorFeatureFormat.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for tensorCategory - * - * @see TensorFeatureFormat.Fields#tensorCategory - */ - public boolean hasTensorCategory() { - if (_tensorCategoryField!= null) { - return true; - } - return super._map.containsKey("tensorCategory"); - } - - /** - * Remover for tensorCategory - * - * @see TensorFeatureFormat.Fields#tensorCategory - */ - public void removeTensorCategory() { - super._map.remove("tensorCategory"); - } - - /** - * Getter for tensorCategory - * - * @see TensorFeatureFormat.Fields#tensorCategory - */ - public TensorCategory getTensorCategory(GetMode mode) { - switch (mode) { - case STRICT: - return getTensorCategory(); - case DEFAULT: - case NULL: - if (_tensorCategoryField!= null) { - return _tensorCategoryField; - } else { - Object __rawValue = super._map.get("tensorCategory"); - _tensorCategoryField = DataTemplateUtil.coerceEnumOutput(__rawValue, TensorCategory.class, TensorCategory.$UNKNOWN); - return _tensorCategoryField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for tensorCategory - * - * @return - * Required field. Could be null for partial record. - * @see TensorFeatureFormat.Fields#tensorCategory - */ - @Nonnull - public TensorCategory getTensorCategory() { - if (_tensorCategoryField!= null) { - return _tensorCategoryField; - } else { - Object __rawValue = super._map.get("tensorCategory"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("tensorCategory"); - } - _tensorCategoryField = DataTemplateUtil.coerceEnumOutput(__rawValue, TensorCategory.class, TensorCategory.$UNKNOWN); - return _tensorCategoryField; - } - } - - /** - * Setter for tensorCategory - * - * @see TensorFeatureFormat.Fields#tensorCategory - */ - public TensorFeatureFormat setTensorCategory(TensorCategory value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setTensorCategory(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field tensorCategory of com.linkedin.feathr.compute.TensorFeatureFormat"); - } else { - CheckedUtil.putWithoutChecking(super._map, "tensorCategory", value.name()); - _tensorCategoryField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeTensorCategory(); - } else { - CheckedUtil.putWithoutChecking(super._map, "tensorCategory", value.name()); - _tensorCategoryField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "tensorCategory", value.name()); - _tensorCategoryField = value; - } - break; - } - return this; - } - - /** - * Setter for tensorCategory - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TensorFeatureFormat.Fields#tensorCategory - */ - public TensorFeatureFormat setTensorCategory( - @Nonnull - TensorCategory value) { - if (value == null) { - throw new NullPointerException("Cannot set field tensorCategory of com.linkedin.feathr.compute.TensorFeatureFormat to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "tensorCategory", value.name()); - _tensorCategoryField = value; - } - return this; - } - - /** - * Existence checker for valueType - * - * @see TensorFeatureFormat.Fields#valueType - */ - public boolean hasValueType() { - if (_valueTypeField!= null) { - return true; - } - return super._map.containsKey("valueType"); - } - - /** - * Remover for valueType - * - * @see TensorFeatureFormat.Fields#valueType - */ - public void removeValueType() { - super._map.remove("valueType"); - } - - /** - * Getter for valueType - * - * @see TensorFeatureFormat.Fields#valueType - */ - public ValueType getValueType(GetMode mode) { - switch (mode) { - case STRICT: - return getValueType(); - case DEFAULT: - case NULL: - if (_valueTypeField!= null) { - return _valueTypeField; - } else { - Object __rawValue = super._map.get("valueType"); - _valueTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, ValueType.class, ValueType.$UNKNOWN); - return _valueTypeField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for valueType - * - * @return - * Required field. Could be null for partial record. - * @see TensorFeatureFormat.Fields#valueType - */ - @Nonnull - public ValueType getValueType() { - if (_valueTypeField!= null) { - return _valueTypeField; - } else { - Object __rawValue = super._map.get("valueType"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("valueType"); - } - _valueTypeField = DataTemplateUtil.coerceEnumOutput(__rawValue, ValueType.class, ValueType.$UNKNOWN); - return _valueTypeField; - } - } - - /** - * Setter for valueType - * - * @see TensorFeatureFormat.Fields#valueType - */ - public TensorFeatureFormat setValueType(ValueType value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setValueType(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field valueType of com.linkedin.feathr.compute.TensorFeatureFormat"); - } else { - CheckedUtil.putWithoutChecking(super._map, "valueType", value.name()); - _valueTypeField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeValueType(); - } else { - CheckedUtil.putWithoutChecking(super._map, "valueType", value.name()); - _valueTypeField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "valueType", value.name()); - _valueTypeField = value; - } - break; - } - return this; - } - - /** - * Setter for valueType - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TensorFeatureFormat.Fields#valueType - */ - public TensorFeatureFormat setValueType( - @Nonnull - ValueType value) { - if (value == null) { - throw new NullPointerException("Cannot set field valueType of com.linkedin.feathr.compute.TensorFeatureFormat to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "valueType", value.name()); - _valueTypeField = value; - } - return this; - } - - /** - * Existence checker for dimensions - * - * @see TensorFeatureFormat.Fields#dimensions - */ - public boolean hasDimensions() { - if (_dimensionsField!= null) { - return true; - } - return super._map.containsKey("dimensions"); - } - - /** - * Remover for dimensions - * - * @see TensorFeatureFormat.Fields#dimensions - */ - public void removeDimensions() { - super._map.remove("dimensions"); - } - - /** - * Getter for dimensions - * - * @see TensorFeatureFormat.Fields#dimensions - */ - public DimensionArray getDimensions(GetMode mode) { - switch (mode) { - case STRICT: - return getDimensions(); - case DEFAULT: - case NULL: - if (_dimensionsField!= null) { - return _dimensionsField; - } else { - Object __rawValue = super._map.get("dimensions"); - _dimensionsField = ((__rawValue == null)?null:new DimensionArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _dimensionsField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for dimensions - * - * @return - * Required field. Could be null for partial record. - * @see TensorFeatureFormat.Fields#dimensions - */ - @Nonnull - public DimensionArray getDimensions() { - if (_dimensionsField!= null) { - return _dimensionsField; - } else { - Object __rawValue = super._map.get("dimensions"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("dimensions"); - } - _dimensionsField = ((__rawValue == null)?null:new DimensionArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _dimensionsField; - } - } - - /** - * Setter for dimensions - * - * @see TensorFeatureFormat.Fields#dimensions - */ - public TensorFeatureFormat setDimensions(DimensionArray value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setDimensions(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field dimensions of com.linkedin.feathr.compute.TensorFeatureFormat"); - } else { - CheckedUtil.putWithoutChecking(super._map, "dimensions", value.data()); - _dimensionsField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeDimensions(); - } else { - CheckedUtil.putWithoutChecking(super._map, "dimensions", value.data()); - _dimensionsField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "dimensions", value.data()); - _dimensionsField = value; - } - break; - } - return this; - } - - /** - * Setter for dimensions - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TensorFeatureFormat.Fields#dimensions - */ - public TensorFeatureFormat setDimensions( - @Nonnull - DimensionArray value) { - if (value == null) { - throw new NullPointerException("Cannot set field dimensions of com.linkedin.feathr.compute.TensorFeatureFormat to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "dimensions", value.data()); - _dimensionsField = value; - } - return this; - } - - @Override - public TensorFeatureFormat clone() - throws CloneNotSupportedException - { - TensorFeatureFormat __clone = ((TensorFeatureFormat) super.clone()); - __clone.__changeListener = new TensorFeatureFormat.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public TensorFeatureFormat copy() - throws CloneNotSupportedException - { - TensorFeatureFormat __copy = ((TensorFeatureFormat) super.copy()); - __copy._tensorCategoryField = null; - __copy._valueTypeField = null; - __copy._dimensionsField = null; - __copy.__changeListener = new TensorFeatureFormat.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final TensorFeatureFormat __objectRef; - - private ChangeListener(TensorFeatureFormat reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "tensorCategory": - __objectRef._tensorCategoryField = null; - break; - case "valueType": - __objectRef._valueTypeField = null; - break; - case "dimensions": - __objectRef._dimensionsField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Type of the tensor, for example, dense tensor. - * - */ - public PathSpec tensorCategory() { - return new PathSpec(getPathComponents(), "tensorCategory"); - } - - /** - * Type of the value column. - * - */ - public PathSpec valueType() { - return new PathSpec(getPathComponents(), "valueType"); - } - - /** - * A feature data can have zero or more dimensions (columns that represent keys). - * - */ - public com.linkedin.feathr.compute.DimensionArray.Fields dimensions() { - return new com.linkedin.feathr.compute.DimensionArray.Fields(getPathComponents(), "dimensions"); - } - - /** - * A feature data can have zero or more dimensions (columns that represent keys). - * - */ - public PathSpec dimensions(Integer start, Integer count) { - PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "dimensions"); - if (start!= null) { - arrayPathSpec.setAttribute("start", start); - } - if (count!= null) { - arrayPathSpec.setAttribute("count", count); - } - return arrayPathSpec; - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.DimensionArray.ProjectionMask _dimensionsMask; - - ProjectionMask() { - super(4); - } - - /** - * Type of the tensor, for example, dense tensor. - * - */ - public TensorFeatureFormat.ProjectionMask withTensorCategory() { - getDataMap().put("tensorCategory", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Type of the value column. - * - */ - public TensorFeatureFormat.ProjectionMask withValueType() { - getDataMap().put("valueType", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * A feature data can have zero or more dimensions (columns that represent keys). - * - */ - public TensorFeatureFormat.ProjectionMask withDimensions(Function nestedMask) { - _dimensionsMask = nestedMask.apply(((_dimensionsMask == null)?DimensionArray.createMask():_dimensionsMask)); - getDataMap().put("dimensions", _dimensionsMask.getDataMap()); - return this; - } - - /** - * A feature data can have zero or more dimensions (columns that represent keys). - * - */ - public TensorFeatureFormat.ProjectionMask withDimensions() { - _dimensionsMask = null; - getDataMap().put("dimensions", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * A feature data can have zero or more dimensions (columns that represent keys). - * - */ - public TensorFeatureFormat.ProjectionMask withDimensions(Function nestedMask, Integer start, Integer count) { - _dimensionsMask = nestedMask.apply(((_dimensionsMask == null)?DimensionArray.createMask():_dimensionsMask)); - getDataMap().put("dimensions", _dimensionsMask.getDataMap()); - if (start!= null) { - getDataMap().getDataMap("dimensions").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("dimensions").put("$count", count); - } - return this; - } - - /** - * A feature data can have zero or more dimensions (columns that represent keys). - * - */ - public TensorFeatureFormat.ProjectionMask withDimensions(Integer start, Integer count) { - _dimensionsMask = null; - getDataMap().put("dimensions", new DataMap(3)); - if (start!= null) { - getDataMap().getDataMap("dimensions").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("dimensions").put("$count", count); - } - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Time.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Time.java deleted file mode 100644 index d00b4db2d..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Time.java +++ /dev/null @@ -1,30 +0,0 @@ - -package com.linkedin.feathr.compute; - -import javax.annotation.Generated; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.TyperefDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.TyperefInfo; - - -/** - * Number of milliseconds since midnight, January 1, 1970 UTC. It must be a positive number - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Time.pdl.") -public class Time - extends TyperefInfo -{ - - private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Number of milliseconds since midnight, January 1, 1970 UTC. It must be a positive number*/@compliance=\"NONE\"typeref Time=long", SchemaFormatType.PDL)); - - public Time() { - super(SCHEMA); - } - - public static TyperefDataSchema dataSchema() { - return SCHEMA; - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TimestampCol.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TimestampCol.java deleted file mode 100644 index b884b7ac7..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TimestampCol.java +++ /dev/null @@ -1,401 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * Representation of a timestamp column field - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\TimestampCol.pdl.") -public class TimestampCol - extends RecordTemplate -{ - - private final static TimestampCol.Fields _fields = new TimestampCol.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Representation of a timestamp column field*/record TimestampCol{/**Timestamp column expression.*/expression:string/**Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis*/format:string}", SchemaFormatType.PDL)); - private String _expressionField = null; - private String _formatField = null; - private TimestampCol.ChangeListener __changeListener = new TimestampCol.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Expression = SCHEMA.getField("expression"); - private final static RecordDataSchema.Field FIELD_Format = SCHEMA.getField("format"); - - public TimestampCol() { - super(new DataMap(3, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public TimestampCol(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static TimestampCol.Fields fields() { - return _fields; - } - - public static TimestampCol.ProjectionMask createMask() { - return new TimestampCol.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for expression - * - * @see TimestampCol.Fields#expression - */ - public boolean hasExpression() { - if (_expressionField!= null) { - return true; - } - return super._map.containsKey("expression"); - } - - /** - * Remover for expression - * - * @see TimestampCol.Fields#expression - */ - public void removeExpression() { - super._map.remove("expression"); - } - - /** - * Getter for expression - * - * @see TimestampCol.Fields#expression - */ - public String getExpression(GetMode mode) { - switch (mode) { - case STRICT: - return getExpression(); - case DEFAULT: - case NULL: - if (_expressionField!= null) { - return _expressionField; - } else { - Object __rawValue = super._map.get("expression"); - _expressionField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _expressionField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for expression - * - * @return - * Required field. Could be null for partial record. - * @see TimestampCol.Fields#expression - */ - @Nonnull - public String getExpression() { - if (_expressionField!= null) { - return _expressionField; - } else { - Object __rawValue = super._map.get("expression"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("expression"); - } - _expressionField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _expressionField; - } - } - - /** - * Setter for expression - * - * @see TimestampCol.Fields#expression - */ - public TimestampCol setExpression(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setExpression(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field expression of com.linkedin.feathr.compute.TimestampCol"); - } else { - CheckedUtil.putWithoutChecking(super._map, "expression", value); - _expressionField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeExpression(); - } else { - CheckedUtil.putWithoutChecking(super._map, "expression", value); - _expressionField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "expression", value); - _expressionField = value; - } - break; - } - return this; - } - - /** - * Setter for expression - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TimestampCol.Fields#expression - */ - public TimestampCol setExpression( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field expression of com.linkedin.feathr.compute.TimestampCol to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "expression", value); - _expressionField = value; - } - return this; - } - - /** - * Existence checker for format - * - * @see TimestampCol.Fields#format - */ - public boolean hasFormat() { - if (_formatField!= null) { - return true; - } - return super._map.containsKey("format"); - } - - /** - * Remover for format - * - * @see TimestampCol.Fields#format - */ - public void removeFormat() { - super._map.remove("format"); - } - - /** - * Getter for format - * - * @see TimestampCol.Fields#format - */ - public String getFormat(GetMode mode) { - switch (mode) { - case STRICT: - return getFormat(); - case DEFAULT: - case NULL: - if (_formatField!= null) { - return _formatField; - } else { - Object __rawValue = super._map.get("format"); - _formatField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _formatField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for format - * - * @return - * Required field. Could be null for partial record. - * @see TimestampCol.Fields#format - */ - @Nonnull - public String getFormat() { - if (_formatField!= null) { - return _formatField; - } else { - Object __rawValue = super._map.get("format"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("format"); - } - _formatField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _formatField; - } - } - - /** - * Setter for format - * - * @see TimestampCol.Fields#format - */ - public TimestampCol setFormat(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFormat(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field format of com.linkedin.feathr.compute.TimestampCol"); - } else { - CheckedUtil.putWithoutChecking(super._map, "format", value); - _formatField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeFormat(); - } else { - CheckedUtil.putWithoutChecking(super._map, "format", value); - _formatField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "format", value); - _formatField = value; - } - break; - } - return this; - } - - /** - * Setter for format - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TimestampCol.Fields#format - */ - public TimestampCol setFormat( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field format of com.linkedin.feathr.compute.TimestampCol to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "format", value); - _formatField = value; - } - return this; - } - - @Override - public TimestampCol clone() - throws CloneNotSupportedException - { - TimestampCol __clone = ((TimestampCol) super.clone()); - __clone.__changeListener = new TimestampCol.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public TimestampCol copy() - throws CloneNotSupportedException - { - TimestampCol __copy = ((TimestampCol) super.copy()); - __copy._expressionField = null; - __copy._formatField = null; - __copy.__changeListener = new TimestampCol.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final TimestampCol __objectRef; - - private ChangeListener(TimestampCol reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "expression": - __objectRef._expressionField = null; - break; - case "format": - __objectRef._formatField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Timestamp column expression. - * - */ - public PathSpec expression() { - return new PathSpec(getPathComponents(), "expression"); - } - - /** - * Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis - * - */ - public PathSpec format() { - return new PathSpec(getPathComponents(), "format"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(3); - } - - /** - * Timestamp column expression. - * - */ - public TimestampCol.ProjectionMask withExpression() { - getDataMap().put("expression", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Format of the timestamp, example - yyyy/MM/dd, epoch, epoch_millis - * - */ - public TimestampCol.ProjectionMask withFormat() { - getDataMap().put("format", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Transformation.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Transformation.java deleted file mode 100644 index 56de17b55..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Transformation.java +++ /dev/null @@ -1,1065 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * Representation of a transformation node. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Transformation.pdl.") -public class Transformation - extends RecordTemplate -{ - - private final static Transformation.Fields _fields = new Transformation.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Representation of a transformation node.*/record Transformation includes/**Generic abstraction of a node. All other nodes should derive from this node.*/record AbstractNode{/**The node would be represented by this id.*/id:/**A type ref to int node id*/typeref NodeId=int/**The key for which this node is being requested.\r\nIf this node is a Source node, the engine can use the key to fetch or join the feature.\r\nIf this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but\r\nshould follow the node's inputs. (The core libraries may use the key information in order to optimize the graph,\r\ne.g. it can be used for identifying duplicate sections of the graph that can be pruned.)*/concreteKey:optional/**The key (node) for which the node in question is requested.*/record ConcreteKey{/**Most of the time, this should point to a CONTEXT SOURCE node, e.g. a key in the context called x.\r\nThe main exception would be for a Lookup feature, in which case it would point to another node where the lookup\r\nkey gets computed.*/key:array[NodeId]}}{/**An array of node references which should be considered as input to apply the transformation function.*/inputs:array[/**This is used to refer to a node from another node. It is a combination of a node id and the indices of the keys from the\r\noriginal node array.\r\nFor example, consider:-\r\nanchorA: {\r\n key: [viewerId, vieweeId]\r\n feature: featureA\r\n}\r\nLet us say featureA is evaluated in node 1.\r\nderivation: {\r\n key: [vieweeId, viewerId]\r\n args1: {key: [vieweeId, viewerId], feature: featureA}\r\n definition: args1*2\r\n}\r\nNow, the node reference (to represent args1) would be:\r\n nodeId: 1\r\n keyReference: [1,0] - // Indicates the ordering of the key indices.*/record NodeReference{/**node id of the referring node.*/id:NodeId/**The key references in the keys of the referring node.*/keyReference:array[/**This represents the position of the key in the node which is being referred to. For example, if the original node has a key\r\nlike [x, y], and the keyReference says 1, it is referring to y.*/record KeyReference{/**Position in the original key array*/position:int}]}]/**The transformation function.*/function:/**The transformation function*/record TransformationFunction{/**Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class.\r\n*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**The various attributes required to represent the transformation function are captured in a map format.\r\nFor example, mvel expression or java udf class name*/parameters:optional map[string,string]}/**Feature name here is used so we retain feature name, type, and default values even after graph is resolved.\r\nFeature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias.*/featureName:string/**feature version of the feature*/featureVersion:record FeatureVersion{/**Defines the high level semantic type of a feature. The high level semantic types are supported in early version of feathr before Tensorization and will be kept around until a full transition to Tensor types is completed*/type:/**The high level types associated with a feature. In contrast with TensorFeatureFormat which contains additional metadata about the type of the tensor, this represents the high level semantic types supported by early versions of feathr. See https://iwww.corp.linkedin.com/wiki/cf/display/ENGS/Feature+Representation+and+Feature+Type+System for more detais. TODO - this is expected to be deprecated once the full transition to TensorType is completed*/enum FrameFeatureType{/** Boolean valued feature */BOOLEAN/** Numerically valued feature such as INT, LONG, DOUBLE, etc */NUMERIC/** Represents a feature that consists of a single category (e.g. MOBILE, DESKSTOP) */CATEGORICAL/** Represents a feature that consists of multiple categories (e.g. MOBILE, DESKSTOP) */CATEGORICAL_SET/** Represents a feature in vector format where the the majority of the elements are non-zero */DENSE_VECTOR/** Represents features that has string terms and numeric value*/TERM_VECTOR/** Represents tensor based features. Note: this represents the high level semantic tensor type but does not include the low level tensor format such as category, shape, dimension and value types. The latter are defined as part of the new tensor annotation (via TensorFeatureFormat) or the legacy FML (go/FML).*/TENSOR/** Placeholder for when no types are specified */UNSPECIFIED}=\"UNSPECIFIED\"/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. feathr will make some default assumptions if FeatureFormat is not provided, but this should be considered limited support, and format should be defined for all new features.*/format:optional/**Defines the format of feature data. Feature data is produced by applying transformation on source, in a FeatureAnchor. Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions). Each row defines a single key/value pair, each column can have a different type. For more details, refer to doc: https://docs.google.com/document/d/1D3JZWBwI7sgHrNzkHZwV3YNEHn69lZcl4VfhdHVmDJo/edit#. Currently in feathr, there are two ways to specify Feature formats, one is via Name-Term-Value (NTV) types (eg. NUMERIC, TERM_VECTOR, CATEGORICAL, see go/featuretypes), the other is via FML metadata (Feature Metadata Library, go/fml). For NTV types, there is a conversion path to Quince Tensor via Auto Tensorization. Existing NTV types can be mapped to different combinations of valueType and dimensionTypes in a deterministic manner. Refer to doc: https://docs.google.com/document/d/10bJMYlCixhsghCtyD08FsQaoQdAJMcpGnRyGe64TSr4/edit#. Feature owners can choose to define FML metadata (eg. valType, dimension's type, etc, see go/fml), which will also be converted to Quince Tensor internally. The data model in this class should be able to uniformly represent both cases.*/record TensorFeatureFormat{/**Type of the tensor, for example, dense tensor.*/tensorCategory:/**Supported Tensor categories in feathr and Quince.*/enum TensorCategory{/**Dense tensors store values in a contiguous sequential block of memory where all values are represented.*/DENSE/**Sparse tensor represents a dataset in which most of the entries are zero. It does not store the whole values of the tensor object but stores the non-zero values and the corresponding coordinates of them.*/SPARSE/**Ragged tensors (also known as nested tensors) are similar to dense tensors but have variable-length dimensions.*/RAGGED}/**Type of the value column.*/valueType:/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}/**A feature data can have zero or more dimensions (columns that represent keys).*/dimensions:array[/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (aka dimensions).*/record Dimension{/**Type of the dimension in the tensor. Each dimension can have a different type.*/type:/**Supported dimension types for tensors in Quince and feathr.*/enum DimensionType{/** Long. */LONG/** Integer. */INT/** String. */STRING/** Boolean. */BOOLEAN}/**Size of the dimension in the tensor. If unset, it means the size is unknown and actual size will be determined at runtime.*/shape:optional int}]}/**An optional default value can be provided. In case of missing data or errors occurred while applying transformation on source in FeatureAnchor, the default value will be used to populate feature data.*/defaultValue:optional/**Defines supported types that can be used to represent the value of a feature data. An example usage is specifying feature's default value. It currently starts with scalar types and more complex types can be added along with more use cases.*/typeref FeatureValue=union[boolean,int,long,float,double,string,bytes]}}", SchemaFormatType.PDL)); - private Integer _idField = null; - private ConcreteKey _concreteKeyField = null; - private NodeReferenceArray _inputsField = null; - private TransformationFunction _functionField = null; - private String _featureNameField = null; - private FeatureVersion _featureVersionField = null; - private Transformation.ChangeListener __changeListener = new Transformation.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Id = SCHEMA.getField("id"); - private final static RecordDataSchema.Field FIELD_ConcreteKey = SCHEMA.getField("concreteKey"); - private final static RecordDataSchema.Field FIELD_Inputs = SCHEMA.getField("inputs"); - private final static RecordDataSchema.Field FIELD_Function = SCHEMA.getField("function"); - private final static RecordDataSchema.Field FIELD_FeatureName = SCHEMA.getField("featureName"); - private final static RecordDataSchema.Field FIELD_FeatureVersion = SCHEMA.getField("featureVersion"); - - public Transformation() { - super(new DataMap(8, 0.75F), SCHEMA, 6); - addChangeListener(__changeListener); - } - - public Transformation(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static Transformation.Fields fields() { - return _fields; - } - - public static Transformation.ProjectionMask createMask() { - return new Transformation.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for id - * - * @see Transformation.Fields#id - */ - public boolean hasId() { - if (_idField!= null) { - return true; - } - return super._map.containsKey("id"); - } - - /** - * Remover for id - * - * @see Transformation.Fields#id - */ - public void removeId() { - super._map.remove("id"); - } - - /** - * Getter for id - * - * @see Transformation.Fields#id - */ - public Integer getId(GetMode mode) { - switch (mode) { - case STRICT: - return getId(); - case DEFAULT: - case NULL: - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for id - * - * @return - * Required field. Could be null for partial record. - * @see Transformation.Fields#id - */ - @Nonnull - public Integer getId() { - if (_idField!= null) { - return _idField; - } else { - Object __rawValue = super._map.get("id"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("id"); - } - _idField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _idField; - } - } - - /** - * Setter for id - * - * @see Transformation.Fields#id - */ - public Transformation setId(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setId(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field id of com.linkedin.feathr.compute.Transformation"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeId(); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - break; - } - return this; - } - - /** - * Setter for id - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Transformation.Fields#id - */ - public Transformation setId( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field id of com.linkedin.feathr.compute.Transformation to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - } - return this; - } - - /** - * Setter for id - * - * @see Transformation.Fields#id - */ - public Transformation setId(int value) { - CheckedUtil.putWithoutChecking(super._map, "id", DataTemplateUtil.coerceIntInput(value)); - _idField = value; - return this; - } - - /** - * Existence checker for concreteKey - * - * @see Transformation.Fields#concreteKey - */ - public boolean hasConcreteKey() { - if (_concreteKeyField!= null) { - return true; - } - return super._map.containsKey("concreteKey"); - } - - /** - * Remover for concreteKey - * - * @see Transformation.Fields#concreteKey - */ - public void removeConcreteKey() { - super._map.remove("concreteKey"); - } - - /** - * Getter for concreteKey - * - * @see Transformation.Fields#concreteKey - */ - public ConcreteKey getConcreteKey(GetMode mode) { - return getConcreteKey(); - } - - /** - * Getter for concreteKey - * - * @return - * Optional field. Always check for null. - * @see Transformation.Fields#concreteKey - */ - @Nullable - public ConcreteKey getConcreteKey() { - if (_concreteKeyField!= null) { - return _concreteKeyField; - } else { - Object __rawValue = super._map.get("concreteKey"); - _concreteKeyField = ((__rawValue == null)?null:new ConcreteKey(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _concreteKeyField; - } - } - - /** - * Setter for concreteKey - * - * @see Transformation.Fields#concreteKey - */ - public Transformation setConcreteKey(ConcreteKey value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setConcreteKey(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeConcreteKey(); - } else { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - break; - } - return this; - } - - /** - * Setter for concreteKey - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Transformation.Fields#concreteKey - */ - public Transformation setConcreteKey( - @Nonnull - ConcreteKey value) { - if (value == null) { - throw new NullPointerException("Cannot set field concreteKey of com.linkedin.feathr.compute.Transformation to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "concreteKey", value.data()); - _concreteKeyField = value; - } - return this; - } - - /** - * Existence checker for inputs - * - * @see Transformation.Fields#inputs - */ - public boolean hasInputs() { - if (_inputsField!= null) { - return true; - } - return super._map.containsKey("inputs"); - } - - /** - * Remover for inputs - * - * @see Transformation.Fields#inputs - */ - public void removeInputs() { - super._map.remove("inputs"); - } - - /** - * Getter for inputs - * - * @see Transformation.Fields#inputs - */ - public NodeReferenceArray getInputs(GetMode mode) { - switch (mode) { - case STRICT: - return getInputs(); - case DEFAULT: - case NULL: - if (_inputsField!= null) { - return _inputsField; - } else { - Object __rawValue = super._map.get("inputs"); - _inputsField = ((__rawValue == null)?null:new NodeReferenceArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _inputsField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for inputs - * - * @return - * Required field. Could be null for partial record. - * @see Transformation.Fields#inputs - */ - @Nonnull - public NodeReferenceArray getInputs() { - if (_inputsField!= null) { - return _inputsField; - } else { - Object __rawValue = super._map.get("inputs"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("inputs"); - } - _inputsField = ((__rawValue == null)?null:new NodeReferenceArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _inputsField; - } - } - - /** - * Setter for inputs - * - * @see Transformation.Fields#inputs - */ - public Transformation setInputs(NodeReferenceArray value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setInputs(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field inputs of com.linkedin.feathr.compute.Transformation"); - } else { - CheckedUtil.putWithoutChecking(super._map, "inputs", value.data()); - _inputsField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeInputs(); - } else { - CheckedUtil.putWithoutChecking(super._map, "inputs", value.data()); - _inputsField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "inputs", value.data()); - _inputsField = value; - } - break; - } - return this; - } - - /** - * Setter for inputs - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Transformation.Fields#inputs - */ - public Transformation setInputs( - @Nonnull - NodeReferenceArray value) { - if (value == null) { - throw new NullPointerException("Cannot set field inputs of com.linkedin.feathr.compute.Transformation to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "inputs", value.data()); - _inputsField = value; - } - return this; - } - - /** - * Existence checker for function - * - * @see Transformation.Fields#function - */ - public boolean hasFunction() { - if (_functionField!= null) { - return true; - } - return super._map.containsKey("function"); - } - - /** - * Remover for function - * - * @see Transformation.Fields#function - */ - public void removeFunction() { - super._map.remove("function"); - } - - /** - * Getter for function - * - * @see Transformation.Fields#function - */ - public TransformationFunction getFunction(GetMode mode) { - switch (mode) { - case STRICT: - return getFunction(); - case DEFAULT: - case NULL: - if (_functionField!= null) { - return _functionField; - } else { - Object __rawValue = super._map.get("function"); - _functionField = ((__rawValue == null)?null:new TransformationFunction(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _functionField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for function - * - * @return - * Required field. Could be null for partial record. - * @see Transformation.Fields#function - */ - @Nonnull - public TransformationFunction getFunction() { - if (_functionField!= null) { - return _functionField; - } else { - Object __rawValue = super._map.get("function"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("function"); - } - _functionField = ((__rawValue == null)?null:new TransformationFunction(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _functionField; - } - } - - /** - * Setter for function - * - * @see Transformation.Fields#function - */ - public Transformation setFunction(TransformationFunction value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFunction(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field function of com.linkedin.feathr.compute.Transformation"); - } else { - CheckedUtil.putWithoutChecking(super._map, "function", value.data()); - _functionField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeFunction(); - } else { - CheckedUtil.putWithoutChecking(super._map, "function", value.data()); - _functionField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "function", value.data()); - _functionField = value; - } - break; - } - return this; - } - - /** - * Setter for function - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Transformation.Fields#function - */ - public Transformation setFunction( - @Nonnull - TransformationFunction value) { - if (value == null) { - throw new NullPointerException("Cannot set field function of com.linkedin.feathr.compute.Transformation to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "function", value.data()); - _functionField = value; - } - return this; - } - - /** - * Existence checker for featureName - * - * @see Transformation.Fields#featureName - */ - public boolean hasFeatureName() { - if (_featureNameField!= null) { - return true; - } - return super._map.containsKey("featureName"); - } - - /** - * Remover for featureName - * - * @see Transformation.Fields#featureName - */ - public void removeFeatureName() { - super._map.remove("featureName"); - } - - /** - * Getter for featureName - * - * @see Transformation.Fields#featureName - */ - public String getFeatureName(GetMode mode) { - switch (mode) { - case STRICT: - return getFeatureName(); - case DEFAULT: - case NULL: - if (_featureNameField!= null) { - return _featureNameField; - } else { - Object __rawValue = super._map.get("featureName"); - _featureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _featureNameField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for featureName - * - * @return - * Required field. Could be null for partial record. - * @see Transformation.Fields#featureName - */ - @Nonnull - public String getFeatureName() { - if (_featureNameField!= null) { - return _featureNameField; - } else { - Object __rawValue = super._map.get("featureName"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("featureName"); - } - _featureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _featureNameField; - } - } - - /** - * Setter for featureName - * - * @see Transformation.Fields#featureName - */ - public Transformation setFeatureName(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFeatureName(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field featureName of com.linkedin.feathr.compute.Transformation"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureName", value); - _featureNameField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeFeatureName(); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureName", value); - _featureNameField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "featureName", value); - _featureNameField = value; - } - break; - } - return this; - } - - /** - * Setter for featureName - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Transformation.Fields#featureName - */ - public Transformation setFeatureName( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field featureName of com.linkedin.feathr.compute.Transformation to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureName", value); - _featureNameField = value; - } - return this; - } - - /** - * Existence checker for featureVersion - * - * @see Transformation.Fields#featureVersion - */ - public boolean hasFeatureVersion() { - if (_featureVersionField!= null) { - return true; - } - return super._map.containsKey("featureVersion"); - } - - /** - * Remover for featureVersion - * - * @see Transformation.Fields#featureVersion - */ - public void removeFeatureVersion() { - super._map.remove("featureVersion"); - } - - /** - * Getter for featureVersion - * - * @see Transformation.Fields#featureVersion - */ - public FeatureVersion getFeatureVersion(GetMode mode) { - switch (mode) { - case STRICT: - return getFeatureVersion(); - case DEFAULT: - case NULL: - if (_featureVersionField!= null) { - return _featureVersionField; - } else { - Object __rawValue = super._map.get("featureVersion"); - _featureVersionField = ((__rawValue == null)?null:new FeatureVersion(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _featureVersionField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for featureVersion - * - * @return - * Required field. Could be null for partial record. - * @see Transformation.Fields#featureVersion - */ - @Nonnull - public FeatureVersion getFeatureVersion() { - if (_featureVersionField!= null) { - return _featureVersionField; - } else { - Object __rawValue = super._map.get("featureVersion"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("featureVersion"); - } - _featureVersionField = ((__rawValue == null)?null:new FeatureVersion(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _featureVersionField; - } - } - - /** - * Setter for featureVersion - * - * @see Transformation.Fields#featureVersion - */ - public Transformation setFeatureVersion(FeatureVersion value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFeatureVersion(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field featureVersion of com.linkedin.feathr.compute.Transformation"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); - _featureVersionField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeFeatureVersion(); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); - _featureVersionField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); - _featureVersionField = value; - } - break; - } - return this; - } - - /** - * Setter for featureVersion - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Transformation.Fields#featureVersion - */ - public Transformation setFeatureVersion( - @Nonnull - FeatureVersion value) { - if (value == null) { - throw new NullPointerException("Cannot set field featureVersion of com.linkedin.feathr.compute.Transformation to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureVersion", value.data()); - _featureVersionField = value; - } - return this; - } - - @Override - public Transformation clone() - throws CloneNotSupportedException - { - Transformation __clone = ((Transformation) super.clone()); - __clone.__changeListener = new Transformation.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public Transformation copy() - throws CloneNotSupportedException - { - Transformation __copy = ((Transformation) super.copy()); - __copy._featureNameField = null; - __copy._inputsField = null; - __copy._functionField = null; - __copy._idField = null; - __copy._concreteKeyField = null; - __copy._featureVersionField = null; - __copy.__changeListener = new Transformation.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final Transformation __objectRef; - - private ChangeListener(Transformation reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "featureName": - __objectRef._featureNameField = null; - break; - case "inputs": - __objectRef._inputsField = null; - break; - case "function": - __objectRef._functionField = null; - break; - case "id": - __objectRef._idField = null; - break; - case "concreteKey": - __objectRef._concreteKeyField = null; - break; - case "featureVersion": - __objectRef._featureVersionField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The node would be represented by this id. - * - */ - public PathSpec id() { - return new PathSpec(getPathComponents(), "id"); - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public com.linkedin.feathr.compute.ConcreteKey.Fields concreteKey() { - return new com.linkedin.feathr.compute.ConcreteKey.Fields(getPathComponents(), "concreteKey"); - } - - /** - * An array of node references which should be considered as input to apply the transformation function. - * - */ - public com.linkedin.feathr.compute.NodeReferenceArray.Fields inputs() { - return new com.linkedin.feathr.compute.NodeReferenceArray.Fields(getPathComponents(), "inputs"); - } - - /** - * An array of node references which should be considered as input to apply the transformation function. - * - */ - public PathSpec inputs(Integer start, Integer count) { - PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "inputs"); - if (start!= null) { - arrayPathSpec.setAttribute("start", start); - } - if (count!= null) { - arrayPathSpec.setAttribute("count", count); - } - return arrayPathSpec; - } - - /** - * The transformation function. - * - */ - public com.linkedin.feathr.compute.TransformationFunction.Fields function() { - return new com.linkedin.feathr.compute.TransformationFunction.Fields(getPathComponents(), "function"); - } - - /** - * Feature name here is used so we retain feature name, type, and default values even after graph is resolved. - * Feature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias. - * - */ - public PathSpec featureName() { - return new PathSpec(getPathComponents(), "featureName"); - } - - /** - * feature version of the feature - * - */ - public com.linkedin.feathr.compute.FeatureVersion.Fields featureVersion() { - return new com.linkedin.feathr.compute.FeatureVersion.Fields(getPathComponents(), "featureVersion"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.compute.ConcreteKey.ProjectionMask _concreteKeyMask; - private com.linkedin.feathr.compute.NodeReferenceArray.ProjectionMask _inputsMask; - private com.linkedin.feathr.compute.TransformationFunction.ProjectionMask _functionMask; - private com.linkedin.feathr.compute.FeatureVersion.ProjectionMask _featureVersionMask; - - ProjectionMask() { - super(8); - } - - /** - * The node would be represented by this id. - * - */ - public Transformation.ProjectionMask withId() { - getDataMap().put("id", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public Transformation.ProjectionMask withConcreteKey(Function nestedMask) { - _concreteKeyMask = nestedMask.apply(((_concreteKeyMask == null)?ConcreteKey.createMask():_concreteKeyMask)); - getDataMap().put("concreteKey", _concreteKeyMask.getDataMap()); - return this; - } - - /** - * The key for which this node is being requested. - * If this node is a Source node, the engine can use the key to fetch or join the feature. - * If this node is NOT a Source node, the engine should NOT use the key to determine fetch/join behavior, but - * should follow the node's inputs. (The core libraries may use the key information in order to optimize the graph, - * e.g. it can be used for identifying duplicate sections of the graph that can be pruned.) - * - */ - public Transformation.ProjectionMask withConcreteKey() { - _concreteKeyMask = null; - getDataMap().put("concreteKey", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * An array of node references which should be considered as input to apply the transformation function. - * - */ - public Transformation.ProjectionMask withInputs(Function nestedMask) { - _inputsMask = nestedMask.apply(((_inputsMask == null)?NodeReferenceArray.createMask():_inputsMask)); - getDataMap().put("inputs", _inputsMask.getDataMap()); - return this; - } - - /** - * An array of node references which should be considered as input to apply the transformation function. - * - */ - public Transformation.ProjectionMask withInputs() { - _inputsMask = null; - getDataMap().put("inputs", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * An array of node references which should be considered as input to apply the transformation function. - * - */ - public Transformation.ProjectionMask withInputs(Function nestedMask, Integer start, Integer count) { - _inputsMask = nestedMask.apply(((_inputsMask == null)?NodeReferenceArray.createMask():_inputsMask)); - getDataMap().put("inputs", _inputsMask.getDataMap()); - if (start!= null) { - getDataMap().getDataMap("inputs").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("inputs").put("$count", count); - } - return this; - } - - /** - * An array of node references which should be considered as input to apply the transformation function. - * - */ - public Transformation.ProjectionMask withInputs(Integer start, Integer count) { - _inputsMask = null; - getDataMap().put("inputs", new DataMap(3)); - if (start!= null) { - getDataMap().getDataMap("inputs").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("inputs").put("$count", count); - } - return this; - } - - /** - * The transformation function. - * - */ - public Transformation.ProjectionMask withFunction(Function nestedMask) { - _functionMask = nestedMask.apply(((_functionMask == null)?TransformationFunction.createMask():_functionMask)); - getDataMap().put("function", _functionMask.getDataMap()); - return this; - } - - /** - * The transformation function. - * - */ - public Transformation.ProjectionMask withFunction() { - _functionMask = null; - getDataMap().put("function", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Feature name here is used so we retain feature name, type, and default values even after graph is resolved. - * Feature name here is also used for feature aliasing in the case where TransformationFunction is feature_alias. - * - */ - public Transformation.ProjectionMask withFeatureName() { - getDataMap().put("featureName", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * feature version of the feature - * - */ - public Transformation.ProjectionMask withFeatureVersion(Function nestedMask) { - _featureVersionMask = nestedMask.apply(((_featureVersionMask == null)?FeatureVersion.createMask():_featureVersionMask)); - getDataMap().put("featureVersion", _featureVersionMask.getDataMap()); - return this; - } - - /** - * feature version of the feature - * - */ - public Transformation.ProjectionMask withFeatureVersion() { - _featureVersionMask = null; - getDataMap().put("featureVersion", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TransformationFunction.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TransformationFunction.java deleted file mode 100644 index 1a44e0193..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/TransformationFunction.java +++ /dev/null @@ -1,384 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; -import com.linkedin.data.template.StringMap; - - -/** - * The transformation function - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\TransformationFunction.pdl.") -public class TransformationFunction - extends RecordTemplate -{ - - private final static TransformationFunction.Fields _fields = new TransformationFunction.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**The transformation function*/record TransformationFunction{/**Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class.\r\n*/operator:/**operator id to set an operator. It can be referring to an mvel expression, sql expression or a java udf.*/typeref OperatorId=string/**The various attributes required to represent the transformation function are captured in a map format.\r\nFor example, mvel expression or java udf class name*/parameters:optional map[string,string]}", SchemaFormatType.PDL)); - private String _operatorField = null; - private StringMap _parametersField = null; - private TransformationFunction.ChangeListener __changeListener = new TransformationFunction.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Operator = SCHEMA.getField("operator"); - private final static RecordDataSchema.Field FIELD_Parameters = SCHEMA.getField("parameters"); - - public TransformationFunction() { - super(new DataMap(3, 0.75F), SCHEMA, 2); - addChangeListener(__changeListener); - } - - public TransformationFunction(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static TransformationFunction.Fields fields() { - return _fields; - } - - public static TransformationFunction.ProjectionMask createMask() { - return new TransformationFunction.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for operator - * - * @see TransformationFunction.Fields#operator - */ - public boolean hasOperator() { - if (_operatorField!= null) { - return true; - } - return super._map.containsKey("operator"); - } - - /** - * Remover for operator - * - * @see TransformationFunction.Fields#operator - */ - public void removeOperator() { - super._map.remove("operator"); - } - - /** - * Getter for operator - * - * @see TransformationFunction.Fields#operator - */ - public String getOperator(GetMode mode) { - switch (mode) { - case STRICT: - return getOperator(); - case DEFAULT: - case NULL: - if (_operatorField!= null) { - return _operatorField; - } else { - Object __rawValue = super._map.get("operator"); - _operatorField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _operatorField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for operator - * - * @return - * Required field. Could be null for partial record. - * @see TransformationFunction.Fields#operator - */ - @Nonnull - public String getOperator() { - if (_operatorField!= null) { - return _operatorField; - } else { - Object __rawValue = super._map.get("operator"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("operator"); - } - _operatorField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _operatorField; - } - } - - /** - * Setter for operator - * - * @see TransformationFunction.Fields#operator - */ - public TransformationFunction setOperator(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setOperator(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field operator of com.linkedin.feathr.compute.TransformationFunction"); - } else { - CheckedUtil.putWithoutChecking(super._map, "operator", value); - _operatorField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeOperator(); - } else { - CheckedUtil.putWithoutChecking(super._map, "operator", value); - _operatorField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "operator", value); - _operatorField = value; - } - break; - } - return this; - } - - /** - * Setter for operator - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TransformationFunction.Fields#operator - */ - public TransformationFunction setOperator( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field operator of com.linkedin.feathr.compute.TransformationFunction to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "operator", value); - _operatorField = value; - } - return this; - } - - /** - * Existence checker for parameters - * - * @see TransformationFunction.Fields#parameters - */ - public boolean hasParameters() { - if (_parametersField!= null) { - return true; - } - return super._map.containsKey("parameters"); - } - - /** - * Remover for parameters - * - * @see TransformationFunction.Fields#parameters - */ - public void removeParameters() { - super._map.remove("parameters"); - } - - /** - * Getter for parameters - * - * @see TransformationFunction.Fields#parameters - */ - public StringMap getParameters(GetMode mode) { - return getParameters(); - } - - /** - * Getter for parameters - * - * @return - * Optional field. Always check for null. - * @see TransformationFunction.Fields#parameters - */ - @Nullable - public StringMap getParameters() { - if (_parametersField!= null) { - return _parametersField; - } else { - Object __rawValue = super._map.get("parameters"); - _parametersField = ((__rawValue == null)?null:new StringMap(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _parametersField; - } - } - - /** - * Setter for parameters - * - * @see TransformationFunction.Fields#parameters - */ - public TransformationFunction setParameters(StringMap value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setParameters(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeParameters(); - } else { - CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); - _parametersField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); - _parametersField = value; - } - break; - } - return this; - } - - /** - * Setter for parameters - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TransformationFunction.Fields#parameters - */ - public TransformationFunction setParameters( - @Nonnull - StringMap value) { - if (value == null) { - throw new NullPointerException("Cannot set field parameters of com.linkedin.feathr.compute.TransformationFunction to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); - _parametersField = value; - } - return this; - } - - @Override - public TransformationFunction clone() - throws CloneNotSupportedException - { - TransformationFunction __clone = ((TransformationFunction) super.clone()); - __clone.__changeListener = new TransformationFunction.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public TransformationFunction copy() - throws CloneNotSupportedException - { - TransformationFunction __copy = ((TransformationFunction) super.copy()); - __copy._parametersField = null; - __copy._operatorField = null; - __copy.__changeListener = new TransformationFunction.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final TransformationFunction __objectRef; - - private ChangeListener(TransformationFunction reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "parameters": - __objectRef._parametersField = null; - break; - case "operator": - __objectRef._operatorField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class. - * - * - */ - public PathSpec operator() { - return new PathSpec(getPathComponents(), "operator"); - } - - /** - * The various attributes required to represent the transformation function are captured in a map format. - * For example, mvel expression or java udf class name - * - */ - public PathSpec parameters() { - return new PathSpec(getPathComponents(), "parameters"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(3); - } - - /** - * Indicates the operator type to be used here. The various different operators supported are in [[Operators]] class. - * - * - */ - public TransformationFunction.ProjectionMask withOperator() { - getDataMap().put("operator", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * The various attributes required to represent the transformation function are captured in a map format. - * For example, mvel expression or java udf class name - * - */ - public TransformationFunction.ProjectionMask withParameters() { - getDataMap().put("parameters", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Unit.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Unit.java deleted file mode 100644 index c433dc921..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Unit.java +++ /dev/null @@ -1,48 +0,0 @@ - -package com.linkedin.feathr.compute; - -import javax.annotation.Generated; -import com.linkedin.data.schema.EnumDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; - - -/** - * - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Window.pdl.") -public enum Unit { - - - /** - * A day. - * - */ - DAY, - - /** - * An hour. - * - */ - HOUR, - - /** - * A minute. - * - */ - MINUTE, - - /** - * A second. - * - */ - SECOND, - $UNKNOWN; - private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute,enum Unit{/** A day. */DAY/** An hour. */HOUR/** A minute. */MINUTE/** A second. */SECOND}", SchemaFormatType.PDL)); - - public static EnumDataSchema dataSchema() { - return SCHEMA; - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/UserDefinedFunction.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/UserDefinedFunction.java deleted file mode 100644 index 40035e4b8..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/UserDefinedFunction.java +++ /dev/null @@ -1,407 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; -import com.linkedin.data.template.StringMap; - - -/** - * User defined function that can be used in feature extraction or derivation. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\UserDefinedFunction.pdl.") -public class UserDefinedFunction - extends RecordTemplate -{ - - private final static UserDefinedFunction.Fields _fields = new UserDefinedFunction.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**User defined function that can be used in feature extraction or derivation.*/record UserDefinedFunction{/**Reference to the class that implements the user defined function.*/clazz:string/**Some UserDefinedFunction requires additional custom parameters. This field defines the custom parameters of the user defined function, represented as a map of string to json blob. The key is the parameter name, and the value is the parameter value represented as a json blob. For example, the parameters may look like: { param1 : [\"waterlooCompany_terms_hashed\", \"waterlooCompany_values\"], param2 : \"com.linkedin.quasar.encoding.SomeEncodingClass\u00e2\u20ac\ufffd } feathr will be responsible of parsing the parameters map into a CustomParameters class defined by application: public class CustomParameters { List param1; String param2; } CustomParameters will be used in the constructor of the UserDefinedFunction.*/parameters:map[string,string]={}}", SchemaFormatType.PDL)); - private String _clazzField = null; - private StringMap _parametersField = null; - private UserDefinedFunction.ChangeListener __changeListener = new UserDefinedFunction.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Clazz = SCHEMA.getField("clazz"); - private final static RecordDataSchema.Field FIELD_Parameters = SCHEMA.getField("parameters"); - private final static StringMap DEFAULT_Parameters; - - static { - DEFAULT_Parameters = ((FIELD_Parameters.getDefault() == null)?null:new StringMap(DataTemplateUtil.castOrThrow(FIELD_Parameters.getDefault(), DataMap.class))); - } - - public UserDefinedFunction() { - super(new DataMap(3, 0.75F), SCHEMA, 2); - addChangeListener(__changeListener); - } - - public UserDefinedFunction(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UserDefinedFunction.Fields fields() { - return _fields; - } - - public static UserDefinedFunction.ProjectionMask createMask() { - return new UserDefinedFunction.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for clazz - * - * @see UserDefinedFunction.Fields#clazz - */ - public boolean hasClazz() { - if (_clazzField!= null) { - return true; - } - return super._map.containsKey("clazz"); - } - - /** - * Remover for clazz - * - * @see UserDefinedFunction.Fields#clazz - */ - public void removeClazz() { - super._map.remove("clazz"); - } - - /** - * Getter for clazz - * - * @see UserDefinedFunction.Fields#clazz - */ - public String getClazz(GetMode mode) { - switch (mode) { - case STRICT: - return getClazz(); - case DEFAULT: - case NULL: - if (_clazzField!= null) { - return _clazzField; - } else { - Object __rawValue = super._map.get("clazz"); - _clazzField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _clazzField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for clazz - * - * @return - * Required field. Could be null for partial record. - * @see UserDefinedFunction.Fields#clazz - */ - @Nonnull - public String getClazz() { - if (_clazzField!= null) { - return _clazzField; - } else { - Object __rawValue = super._map.get("clazz"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("clazz"); - } - _clazzField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _clazzField; - } - } - - /** - * Setter for clazz - * - * @see UserDefinedFunction.Fields#clazz - */ - public UserDefinedFunction setClazz(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setClazz(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field clazz of com.linkedin.feathr.compute.UserDefinedFunction"); - } else { - CheckedUtil.putWithoutChecking(super._map, "clazz", value); - _clazzField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeClazz(); - } else { - CheckedUtil.putWithoutChecking(super._map, "clazz", value); - _clazzField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "clazz", value); - _clazzField = value; - } - break; - } - return this; - } - - /** - * Setter for clazz - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see UserDefinedFunction.Fields#clazz - */ - public UserDefinedFunction setClazz( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field clazz of com.linkedin.feathr.compute.UserDefinedFunction to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "clazz", value); - _clazzField = value; - } - return this; - } - - /** - * Existence checker for parameters - * - * @see UserDefinedFunction.Fields#parameters - */ - public boolean hasParameters() { - if (_parametersField!= null) { - return true; - } - return super._map.containsKey("parameters"); - } - - /** - * Remover for parameters - * - * @see UserDefinedFunction.Fields#parameters - */ - public void removeParameters() { - super._map.remove("parameters"); - } - - /** - * Getter for parameters - * - * @see UserDefinedFunction.Fields#parameters - */ - public StringMap getParameters(GetMode mode) { - switch (mode) { - case STRICT: - case DEFAULT: - return getParameters(); - case NULL: - if (_parametersField!= null) { - return _parametersField; - } else { - Object __rawValue = super._map.get("parameters"); - _parametersField = ((__rawValue == null)?null:new StringMap(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _parametersField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for parameters - * - * @return - * Required field. Could be null for partial record. - * @see UserDefinedFunction.Fields#parameters - */ - @Nonnull - public StringMap getParameters() { - if (_parametersField!= null) { - return _parametersField; - } else { - Object __rawValue = super._map.get("parameters"); - if (__rawValue == null) { - return DEFAULT_Parameters; - } - _parametersField = ((__rawValue == null)?null:new StringMap(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _parametersField; - } - } - - /** - * Setter for parameters - * - * @see UserDefinedFunction.Fields#parameters - */ - public UserDefinedFunction setParameters(StringMap value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setParameters(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field parameters of com.linkedin.feathr.compute.UserDefinedFunction"); - } else { - CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); - _parametersField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeParameters(); - } else { - CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); - _parametersField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); - _parametersField = value; - } - break; - } - return this; - } - - /** - * Setter for parameters - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see UserDefinedFunction.Fields#parameters - */ - public UserDefinedFunction setParameters( - @Nonnull - StringMap value) { - if (value == null) { - throw new NullPointerException("Cannot set field parameters of com.linkedin.feathr.compute.UserDefinedFunction to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "parameters", value.data()); - _parametersField = value; - } - return this; - } - - @Override - public UserDefinedFunction clone() - throws CloneNotSupportedException - { - UserDefinedFunction __clone = ((UserDefinedFunction) super.clone()); - __clone.__changeListener = new UserDefinedFunction.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public UserDefinedFunction copy() - throws CloneNotSupportedException - { - UserDefinedFunction __copy = ((UserDefinedFunction) super.copy()); - __copy._clazzField = null; - __copy._parametersField = null; - __copy.__changeListener = new UserDefinedFunction.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final UserDefinedFunction __objectRef; - - private ChangeListener(UserDefinedFunction reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "clazz": - __objectRef._clazzField = null; - break; - case "parameters": - __objectRef._parametersField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Reference to the class that implements the user defined function. - * - */ - public PathSpec clazz() { - return new PathSpec(getPathComponents(), "clazz"); - } - - /** - * Some UserDefinedFunction requires additional custom parameters. This field defines the custom parameters of the user defined function, represented as a map of string to json blob. The key is the parameter name, and the value is the parameter value represented as a json blob. For example, the parameters may look like: { param1 : ["waterlooCompany_terms_hashed", "waterlooCompany_values"], param2 : "com.linkedin.quasar.encoding.SomeEncodingClassâ€\ufffd } feathr will be responsible of parsing the parameters map into a CustomParameters class defined by application: public class CustomParameters { List param1; String param2; } CustomParameters will be used in the constructor of the UserDefinedFunction. - * - */ - public PathSpec parameters() { - return new PathSpec(getPathComponents(), "parameters"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(3); - } - - /** - * Reference to the class that implements the user defined function. - * - */ - public UserDefinedFunction.ProjectionMask withClazz() { - getDataMap().put("clazz", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Some UserDefinedFunction requires additional custom parameters. This field defines the custom parameters of the user defined function, represented as a map of string to json blob. The key is the parameter name, and the value is the parameter value represented as a json blob. For example, the parameters may look like: { param1 : ["waterlooCompany_terms_hashed", "waterlooCompany_values"], param2 : "com.linkedin.quasar.encoding.SomeEncodingClassâ€\ufffd } feathr will be responsible of parsing the parameters map into a CustomParameters class defined by application: public class CustomParameters { List param1; String param2; } CustomParameters will be used in the constructor of the UserDefinedFunction. - * - */ - public UserDefinedFunction.ProjectionMask withParameters() { - getDataMap().put("parameters", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ValueType.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ValueType.java deleted file mode 100644 index f29229bd9..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/ValueType.java +++ /dev/null @@ -1,66 +0,0 @@ - -package com.linkedin.feathr.compute; - -import javax.annotation.Generated; -import com.linkedin.data.schema.EnumDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; - - -/** - * Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\ValueType.pdl.") -public enum ValueType { - - - /** - * Integer. - * - */ - INT, - - /** - * Long. - * - */ - LONG, - - /** - * Float. - * - */ - FLOAT, - - /** - * Double. - * - */ - DOUBLE, - - /** - * String. - * - */ - STRING, - - /** - * Boolean. - * - */ - BOOLEAN, - - /** - * Byte array. - * - */ - BYTES, - $UNKNOWN; - private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Tensor is used to represent feature data. A tensor is a generalization of vectors and matrices to potentially higher dimensions. In Quince Tensor specifically, the last column is designated as the value, and the rest of the columns are keys (or dimensions); Each row defines a single key/value pair. This enum defines supported value types for tensors in Quince and feathr.*/enum ValueType{/** Integer. */INT/** Long. */LONG/** Float. */FLOAT/** Double. */DOUBLE/** String. */STRING/** Boolean. */BOOLEAN/** Byte array. */BYTES}", SchemaFormatType.PDL)); - - public static EnumDataSchema dataSchema() { - return SCHEMA; - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Window.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Window.java deleted file mode 100644 index 83512a2e7..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/compute/Window.java +++ /dev/null @@ -1,412 +0,0 @@ - -package com.linkedin.feathr.compute; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * Represents a time window used in sliding window algorithms. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\compute\\Window.pdl.") -public class Window - extends RecordTemplate -{ - - private final static Window.Fields _fields = new Window.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.compute/**Represents a time window used in sliding window algorithms.*/record Window{/**Represents the duration of the window.*/size:int/**Represents a unit of time.*/unit:enum Unit{/** A day. */DAY/** An hour. */HOUR/** A minute. */MINUTE/** A second. */SECOND}}", SchemaFormatType.PDL)); - private Integer _sizeField = null; - private Unit _unitField = null; - private Window.ChangeListener __changeListener = new Window.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Size = SCHEMA.getField("size"); - private final static RecordDataSchema.Field FIELD_Unit = SCHEMA.getField("unit"); - - public Window() { - super(new DataMap(3, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public Window(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static Window.Fields fields() { - return _fields; - } - - public static Window.ProjectionMask createMask() { - return new Window.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for size - * - * @see Window.Fields#size - */ - public boolean hasSize() { - if (_sizeField!= null) { - return true; - } - return super._map.containsKey("size"); - } - - /** - * Remover for size - * - * @see Window.Fields#size - */ - public void removeSize() { - super._map.remove("size"); - } - - /** - * Getter for size - * - * @see Window.Fields#size - */ - public Integer getSize(GetMode mode) { - switch (mode) { - case STRICT: - return getSize(); - case DEFAULT: - case NULL: - if (_sizeField!= null) { - return _sizeField; - } else { - Object __rawValue = super._map.get("size"); - _sizeField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _sizeField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for size - * - * @return - * Required field. Could be null for partial record. - * @see Window.Fields#size - */ - @Nonnull - public Integer getSize() { - if (_sizeField!= null) { - return _sizeField; - } else { - Object __rawValue = super._map.get("size"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("size"); - } - _sizeField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _sizeField; - } - } - - /** - * Setter for size - * - * @see Window.Fields#size - */ - public Window setSize(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setSize(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field size of com.linkedin.feathr.compute.Window"); - } else { - CheckedUtil.putWithoutChecking(super._map, "size", DataTemplateUtil.coerceIntInput(value)); - _sizeField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeSize(); - } else { - CheckedUtil.putWithoutChecking(super._map, "size", DataTemplateUtil.coerceIntInput(value)); - _sizeField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "size", DataTemplateUtil.coerceIntInput(value)); - _sizeField = value; - } - break; - } - return this; - } - - /** - * Setter for size - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Window.Fields#size - */ - public Window setSize( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field size of com.linkedin.feathr.compute.Window to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "size", DataTemplateUtil.coerceIntInput(value)); - _sizeField = value; - } - return this; - } - - /** - * Setter for size - * - * @see Window.Fields#size - */ - public Window setSize(int value) { - CheckedUtil.putWithoutChecking(super._map, "size", DataTemplateUtil.coerceIntInput(value)); - _sizeField = value; - return this; - } - - /** - * Existence checker for unit - * - * @see Window.Fields#unit - */ - public boolean hasUnit() { - if (_unitField!= null) { - return true; - } - return super._map.containsKey("unit"); - } - - /** - * Remover for unit - * - * @see Window.Fields#unit - */ - public void removeUnit() { - super._map.remove("unit"); - } - - /** - * Getter for unit - * - * @see Window.Fields#unit - */ - public Unit getUnit(GetMode mode) { - switch (mode) { - case STRICT: - return getUnit(); - case DEFAULT: - case NULL: - if (_unitField!= null) { - return _unitField; - } else { - Object __rawValue = super._map.get("unit"); - _unitField = DataTemplateUtil.coerceEnumOutput(__rawValue, Unit.class, Unit.$UNKNOWN); - return _unitField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for unit - * - * @return - * Required field. Could be null for partial record. - * @see Window.Fields#unit - */ - @Nonnull - public Unit getUnit() { - if (_unitField!= null) { - return _unitField; - } else { - Object __rawValue = super._map.get("unit"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("unit"); - } - _unitField = DataTemplateUtil.coerceEnumOutput(__rawValue, Unit.class, Unit.$UNKNOWN); - return _unitField; - } - } - - /** - * Setter for unit - * - * @see Window.Fields#unit - */ - public Window setUnit(Unit value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setUnit(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field unit of com.linkedin.feathr.compute.Window"); - } else { - CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); - _unitField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeUnit(); - } else { - CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); - _unitField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); - _unitField = value; - } - break; - } - return this; - } - - /** - * Setter for unit - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Window.Fields#unit - */ - public Window setUnit( - @Nonnull - Unit value) { - if (value == null) { - throw new NullPointerException("Cannot set field unit of com.linkedin.feathr.compute.Window to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); - _unitField = value; - } - return this; - } - - @Override - public Window clone() - throws CloneNotSupportedException - { - Window __clone = ((Window) super.clone()); - __clone.__changeListener = new Window.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public Window copy() - throws CloneNotSupportedException - { - Window __copy = ((Window) super.copy()); - __copy._unitField = null; - __copy._sizeField = null; - __copy.__changeListener = new Window.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final Window __objectRef; - - private ChangeListener(Window reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "unit": - __objectRef._unitField = null; - break; - case "size": - __objectRef._sizeField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Represents the duration of the window. - * - */ - public PathSpec size() { - return new PathSpec(getPathComponents(), "size"); - } - - /** - * Represents a unit of time. - * - */ - public PathSpec unit() { - return new PathSpec(getPathComponents(), "unit"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(3); - } - - /** - * Represents the duration of the window. - * - */ - public Window.ProjectionMask withSize() { - getDataMap().put("size", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Represents a unit of time. - * - */ - public Window.ProjectionMask withUnit() { - getDataMap().put("unit", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteDateRange.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteDateRange.java deleted file mode 100644 index f949423f5..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteDateRange.java +++ /dev/null @@ -1,432 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * The absolute date range with start and end date being required fields. - * It accepts a start date and an end date which should be specifiied using the [[Date.pdl]] class. - * absoluteDateRange: { - * startDate: Date(day=1, month=1, year=2020) - * endDate: Date(day=3, month=1, year=2020) - * } - * In this case, the endDate > startDate. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\AbsoluteDateRange.pdl.") -public class AbsoluteDateRange - extends RecordTemplate -{ - - private final static AbsoluteDateRange.Fields _fields = new AbsoluteDateRange.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The absolute date range with start and end date being required fields.\r\nIt accepts a start date and an end date which should be specifiied using the [[Date.pdl]] class.\r\nabsoluteDateRange: {\r\n startDate: Date(day=1, month=1, year=2020)\r\n endDate: Date(day=3, month=1, year=2020)\r\n }\r\n In this case, the endDate > startDate.*/record AbsoluteDateRange{/**start date of the date range, with the start date included in the range.*/startDate:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}/**end date of the date range, with the end date included in the range.*/endDate:Date}", SchemaFormatType.PDL)); - private Date _startDateField = null; - private Date _endDateField = null; - private AbsoluteDateRange.ChangeListener __changeListener = new AbsoluteDateRange.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_StartDate = SCHEMA.getField("startDate"); - private final static RecordDataSchema.Field FIELD_EndDate = SCHEMA.getField("endDate"); - - public AbsoluteDateRange() { - super(new DataMap(3, 0.75F), SCHEMA, 3); - addChangeListener(__changeListener); - } - - public AbsoluteDateRange(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static AbsoluteDateRange.Fields fields() { - return _fields; - } - - public static AbsoluteDateRange.ProjectionMask createMask() { - return new AbsoluteDateRange.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for startDate - * - * @see AbsoluteDateRange.Fields#startDate - */ - public boolean hasStartDate() { - if (_startDateField!= null) { - return true; - } - return super._map.containsKey("startDate"); - } - - /** - * Remover for startDate - * - * @see AbsoluteDateRange.Fields#startDate - */ - public void removeStartDate() { - super._map.remove("startDate"); - } - - /** - * Getter for startDate - * - * @see AbsoluteDateRange.Fields#startDate - */ - public Date getStartDate(GetMode mode) { - switch (mode) { - case STRICT: - return getStartDate(); - case DEFAULT: - case NULL: - if (_startDateField!= null) { - return _startDateField; - } else { - Object __rawValue = super._map.get("startDate"); - _startDateField = ((__rawValue == null)?null:new Date(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _startDateField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for startDate - * - * @return - * Required field. Could be null for partial record. - * @see AbsoluteDateRange.Fields#startDate - */ - @Nonnull - public Date getStartDate() { - if (_startDateField!= null) { - return _startDateField; - } else { - Object __rawValue = super._map.get("startDate"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("startDate"); - } - _startDateField = ((__rawValue == null)?null:new Date(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _startDateField; - } - } - - /** - * Setter for startDate - * - * @see AbsoluteDateRange.Fields#startDate - */ - public AbsoluteDateRange setStartDate(Date value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setStartDate(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field startDate of com.linkedin.feathr.config.join.AbsoluteDateRange"); - } else { - CheckedUtil.putWithoutChecking(super._map, "startDate", value.data()); - _startDateField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeStartDate(); - } else { - CheckedUtil.putWithoutChecking(super._map, "startDate", value.data()); - _startDateField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "startDate", value.data()); - _startDateField = value; - } - break; - } - return this; - } - - /** - * Setter for startDate - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see AbsoluteDateRange.Fields#startDate - */ - public AbsoluteDateRange setStartDate( - @Nonnull - Date value) { - if (value == null) { - throw new NullPointerException("Cannot set field startDate of com.linkedin.feathr.config.join.AbsoluteDateRange to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "startDate", value.data()); - _startDateField = value; - } - return this; - } - - /** - * Existence checker for endDate - * - * @see AbsoluteDateRange.Fields#endDate - */ - public boolean hasEndDate() { - if (_endDateField!= null) { - return true; - } - return super._map.containsKey("endDate"); - } - - /** - * Remover for endDate - * - * @see AbsoluteDateRange.Fields#endDate - */ - public void removeEndDate() { - super._map.remove("endDate"); - } - - /** - * Getter for endDate - * - * @see AbsoluteDateRange.Fields#endDate - */ - public Date getEndDate(GetMode mode) { - switch (mode) { - case STRICT: - return getEndDate(); - case DEFAULT: - case NULL: - if (_endDateField!= null) { - return _endDateField; - } else { - Object __rawValue = super._map.get("endDate"); - _endDateField = ((__rawValue == null)?null:new Date(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _endDateField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for endDate - * - * @return - * Required field. Could be null for partial record. - * @see AbsoluteDateRange.Fields#endDate - */ - @Nonnull - public Date getEndDate() { - if (_endDateField!= null) { - return _endDateField; - } else { - Object __rawValue = super._map.get("endDate"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("endDate"); - } - _endDateField = ((__rawValue == null)?null:new Date(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _endDateField; - } - } - - /** - * Setter for endDate - * - * @see AbsoluteDateRange.Fields#endDate - */ - public AbsoluteDateRange setEndDate(Date value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setEndDate(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field endDate of com.linkedin.feathr.config.join.AbsoluteDateRange"); - } else { - CheckedUtil.putWithoutChecking(super._map, "endDate", value.data()); - _endDateField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeEndDate(); - } else { - CheckedUtil.putWithoutChecking(super._map, "endDate", value.data()); - _endDateField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "endDate", value.data()); - _endDateField = value; - } - break; - } - return this; - } - - /** - * Setter for endDate - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see AbsoluteDateRange.Fields#endDate - */ - public AbsoluteDateRange setEndDate( - @Nonnull - Date value) { - if (value == null) { - throw new NullPointerException("Cannot set field endDate of com.linkedin.feathr.config.join.AbsoluteDateRange to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "endDate", value.data()); - _endDateField = value; - } - return this; - } - - @Override - public AbsoluteDateRange clone() - throws CloneNotSupportedException - { - AbsoluteDateRange __clone = ((AbsoluteDateRange) super.clone()); - __clone.__changeListener = new AbsoluteDateRange.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public AbsoluteDateRange copy() - throws CloneNotSupportedException - { - AbsoluteDateRange __copy = ((AbsoluteDateRange) super.copy()); - __copy._endDateField = null; - __copy._startDateField = null; - __copy.__changeListener = new AbsoluteDateRange.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final AbsoluteDateRange __objectRef; - - private ChangeListener(AbsoluteDateRange reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "endDate": - __objectRef._endDateField = null; - break; - case "startDate": - __objectRef._startDateField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * start date of the date range, with the start date included in the range. - * - */ - public com.linkedin.feathr.config.join.Date.Fields startDate() { - return new com.linkedin.feathr.config.join.Date.Fields(getPathComponents(), "startDate"); - } - - /** - * end date of the date range, with the end date included in the range. - * - */ - public com.linkedin.feathr.config.join.Date.Fields endDate() { - return new com.linkedin.feathr.config.join.Date.Fields(getPathComponents(), "endDate"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.Date.ProjectionMask _startDateMask; - private com.linkedin.feathr.config.join.Date.ProjectionMask _endDateMask; - - ProjectionMask() { - super(3); - } - - /** - * start date of the date range, with the start date included in the range. - * - */ - public AbsoluteDateRange.ProjectionMask withStartDate(Function nestedMask) { - _startDateMask = nestedMask.apply(((_startDateMask == null)?Date.createMask():_startDateMask)); - getDataMap().put("startDate", _startDateMask.getDataMap()); - return this; - } - - /** - * start date of the date range, with the start date included in the range. - * - */ - public AbsoluteDateRange.ProjectionMask withStartDate() { - _startDateMask = null; - getDataMap().put("startDate", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * end date of the date range, with the end date included in the range. - * - */ - public AbsoluteDateRange.ProjectionMask withEndDate(Function nestedMask) { - _endDateMask = nestedMask.apply(((_endDateMask == null)?Date.createMask():_endDateMask)); - getDataMap().put("endDate", _endDateMask.getDataMap()); - return this; - } - - /** - * end date of the date range, with the end date included in the range. - * - */ - public AbsoluteDateRange.ProjectionMask withEndDate() { - _endDateMask = null; - getDataMap().put("endDate", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteTimeRange.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteTimeRange.java deleted file mode 100644 index d51ec4573..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/AbsoluteTimeRange.java +++ /dev/null @@ -1,802 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.DataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.UnionDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; -import com.linkedin.data.template.UnionTemplate; - - -/** - * The absolute time range with start and end time being required fields. - * It accepts a start time and an end time which should be specifiied using the [[Date.pdl]] or the [[HourTime.pdl]] class. - * This model can be used to represent time range in daily or hourly interval. - * absoluteTimeRange: { - * startTime: TimeHour(day=1, month=1, year=2020, hour=13) - * endTime: TimeHour(day=3, month=1, year=2020, hour=2) - * } - * (or) - * absoluteTimeRange: { - * startTime: Date(day=1, month=1, year=2020) - * endTime: Date(day=3, month=1, year=2020) - * } - * endTime and startTime should always have the same granularity, ie - Daily or Hourly. - * endTme > startTime - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\AbsoluteTimeRange.pdl.") -public class AbsoluteTimeRange - extends RecordTemplate -{ - - private final static AbsoluteTimeRange.Fields _fields = new AbsoluteTimeRange.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The absolute time range with start and end time being required fields.\r\nIt accepts a start time and an end time which should be specifiied using the [[Date.pdl]] or the [[HourTime.pdl]] class.\r\nThis model can be used to represent time range in daily or hourly interval.\r\nabsoluteTimeRange: {\r\n startTime: TimeHour(day=1, month=1, year=2020, hour=13)\r\n endTime: TimeHour(day=3, month=1, year=2020, hour=2)\r\n }\r\n(or)\r\nabsoluteTimeRange: {\r\n startTime: Date(day=1, month=1, year=2020)\r\n endTime: Date(day=3, month=1, year=2020)\r\n }\r\nendTime and startTime should always have the same granularity, ie - Daily or Hourly.\r\nendTme > startTime*/record AbsoluteTimeRange{/**start time of the date range, in daily or hourly format with the start date included in the range.*/startTime:union[date:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}hourTime:/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}]/**end date of the date range, in daily or hourly format with the end date included in the range.*/endTime:union[date:Date,hourTime:HourTime]}", SchemaFormatType.PDL)); - private AbsoluteTimeRange.StartTime _startTimeField = null; - private AbsoluteTimeRange.EndTime _endTimeField = null; - private AbsoluteTimeRange.ChangeListener __changeListener = new AbsoluteTimeRange.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_StartTime = SCHEMA.getField("startTime"); - private final static RecordDataSchema.Field FIELD_EndTime = SCHEMA.getField("endTime"); - - public AbsoluteTimeRange() { - super(new DataMap(3, 0.75F), SCHEMA, 3); - addChangeListener(__changeListener); - } - - public AbsoluteTimeRange(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static AbsoluteTimeRange.Fields fields() { - return _fields; - } - - public static AbsoluteTimeRange.ProjectionMask createMask() { - return new AbsoluteTimeRange.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for startTime - * - * @see AbsoluteTimeRange.Fields#startTime - */ - public boolean hasStartTime() { - if (_startTimeField!= null) { - return true; - } - return super._map.containsKey("startTime"); - } - - /** - * Remover for startTime - * - * @see AbsoluteTimeRange.Fields#startTime - */ - public void removeStartTime() { - super._map.remove("startTime"); - } - - /** - * Getter for startTime - * - * @see AbsoluteTimeRange.Fields#startTime - */ - public AbsoluteTimeRange.StartTime getStartTime(GetMode mode) { - switch (mode) { - case STRICT: - return getStartTime(); - case DEFAULT: - case NULL: - if (_startTimeField!= null) { - return _startTimeField; - } else { - Object __rawValue = super._map.get("startTime"); - _startTimeField = ((__rawValue == null)?null:new AbsoluteTimeRange.StartTime(__rawValue)); - return _startTimeField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for startTime - * - * @return - * Required field. Could be null for partial record. - * @see AbsoluteTimeRange.Fields#startTime - */ - @Nonnull - public AbsoluteTimeRange.StartTime getStartTime() { - if (_startTimeField!= null) { - return _startTimeField; - } else { - Object __rawValue = super._map.get("startTime"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("startTime"); - } - _startTimeField = ((__rawValue == null)?null:new AbsoluteTimeRange.StartTime(__rawValue)); - return _startTimeField; - } - } - - /** - * Setter for startTime - * - * @see AbsoluteTimeRange.Fields#startTime - */ - public AbsoluteTimeRange setStartTime(AbsoluteTimeRange.StartTime value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setStartTime(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field startTime of com.linkedin.feathr.config.join.AbsoluteTimeRange"); - } else { - CheckedUtil.putWithoutChecking(super._map, "startTime", value.data()); - _startTimeField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeStartTime(); - } else { - CheckedUtil.putWithoutChecking(super._map, "startTime", value.data()); - _startTimeField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "startTime", value.data()); - _startTimeField = value; - } - break; - } - return this; - } - - /** - * Setter for startTime - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see AbsoluteTimeRange.Fields#startTime - */ - public AbsoluteTimeRange setStartTime( - @Nonnull - AbsoluteTimeRange.StartTime value) { - if (value == null) { - throw new NullPointerException("Cannot set field startTime of com.linkedin.feathr.config.join.AbsoluteTimeRange to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "startTime", value.data()); - _startTimeField = value; - } - return this; - } - - /** - * Existence checker for endTime - * - * @see AbsoluteTimeRange.Fields#endTime - */ - public boolean hasEndTime() { - if (_endTimeField!= null) { - return true; - } - return super._map.containsKey("endTime"); - } - - /** - * Remover for endTime - * - * @see AbsoluteTimeRange.Fields#endTime - */ - public void removeEndTime() { - super._map.remove("endTime"); - } - - /** - * Getter for endTime - * - * @see AbsoluteTimeRange.Fields#endTime - */ - public AbsoluteTimeRange.EndTime getEndTime(GetMode mode) { - switch (mode) { - case STRICT: - return getEndTime(); - case DEFAULT: - case NULL: - if (_endTimeField!= null) { - return _endTimeField; - } else { - Object __rawValue = super._map.get("endTime"); - _endTimeField = ((__rawValue == null)?null:new AbsoluteTimeRange.EndTime(__rawValue)); - return _endTimeField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for endTime - * - * @return - * Required field. Could be null for partial record. - * @see AbsoluteTimeRange.Fields#endTime - */ - @Nonnull - public AbsoluteTimeRange.EndTime getEndTime() { - if (_endTimeField!= null) { - return _endTimeField; - } else { - Object __rawValue = super._map.get("endTime"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("endTime"); - } - _endTimeField = ((__rawValue == null)?null:new AbsoluteTimeRange.EndTime(__rawValue)); - return _endTimeField; - } - } - - /** - * Setter for endTime - * - * @see AbsoluteTimeRange.Fields#endTime - */ - public AbsoluteTimeRange setEndTime(AbsoluteTimeRange.EndTime value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setEndTime(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field endTime of com.linkedin.feathr.config.join.AbsoluteTimeRange"); - } else { - CheckedUtil.putWithoutChecking(super._map, "endTime", value.data()); - _endTimeField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeEndTime(); - } else { - CheckedUtil.putWithoutChecking(super._map, "endTime", value.data()); - _endTimeField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "endTime", value.data()); - _endTimeField = value; - } - break; - } - return this; - } - - /** - * Setter for endTime - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see AbsoluteTimeRange.Fields#endTime - */ - public AbsoluteTimeRange setEndTime( - @Nonnull - AbsoluteTimeRange.EndTime value) { - if (value == null) { - throw new NullPointerException("Cannot set field endTime of com.linkedin.feathr.config.join.AbsoluteTimeRange to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "endTime", value.data()); - _endTimeField = value; - } - return this; - } - - @Override - public AbsoluteTimeRange clone() - throws CloneNotSupportedException - { - AbsoluteTimeRange __clone = ((AbsoluteTimeRange) super.clone()); - __clone.__changeListener = new AbsoluteTimeRange.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public AbsoluteTimeRange copy() - throws CloneNotSupportedException - { - AbsoluteTimeRange __copy = ((AbsoluteTimeRange) super.copy()); - __copy._startTimeField = null; - __copy._endTimeField = null; - __copy.__changeListener = new AbsoluteTimeRange.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final AbsoluteTimeRange __objectRef; - - private ChangeListener(AbsoluteTimeRange reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "startTime": - __objectRef._startTimeField = null; - break; - case "endTime": - __objectRef._endTimeField = null; - break; - } - } - - } - - @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\AbsoluteTimeRange.pdl.") - public static class EndTime - extends UnionTemplate - { - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[date:{namespace com.linkedin.feathr.config.join/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}}hourTime:{namespace com.linkedin.feathr.config.join/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}}]", SchemaFormatType.PDL)); - private com.linkedin.feathr.config.join.Date _dateMember = null; - private com.linkedin.feathr.config.join.HourTime _hourTimeMember = null; - private AbsoluteTimeRange.EndTime.ChangeListener __changeListener = new AbsoluteTimeRange.EndTime.ChangeListener(this); - private final static DataSchema MEMBER_Date = SCHEMA.getTypeByMemberKey("date"); - private final static DataSchema MEMBER_HourTime = SCHEMA.getTypeByMemberKey("hourTime"); - - public EndTime() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public EndTime(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static AbsoluteTimeRange.EndTime createWithDate(com.linkedin.feathr.config.join.Date value) { - AbsoluteTimeRange.EndTime newUnion = new AbsoluteTimeRange.EndTime(); - newUnion.setDate(value); - return newUnion; - } - - public boolean isDate() { - return memberIs("date"); - } - - public com.linkedin.feathr.config.join.Date getDate() { - checkNotNull(); - if (_dateMember!= null) { - return _dateMember; - } - Object __rawValue = super._map.get("date"); - _dateMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.Date(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _dateMember; - } - - public void setDate(com.linkedin.feathr.config.join.Date value) { - checkNotNull(); - super._map.clear(); - _dateMember = value; - CheckedUtil.putWithoutChecking(super._map, "date", value.data()); - } - - public static AbsoluteTimeRange.EndTime createWithHourTime(com.linkedin.feathr.config.join.HourTime value) { - AbsoluteTimeRange.EndTime newUnion = new AbsoluteTimeRange.EndTime(); - newUnion.setHourTime(value); - return newUnion; - } - - public boolean isHourTime() { - return memberIs("hourTime"); - } - - public com.linkedin.feathr.config.join.HourTime getHourTime() { - checkNotNull(); - if (_hourTimeMember!= null) { - return _hourTimeMember; - } - Object __rawValue = super._map.get("hourTime"); - _hourTimeMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.HourTime(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _hourTimeMember; - } - - public void setHourTime(com.linkedin.feathr.config.join.HourTime value) { - checkNotNull(); - super._map.clear(); - _hourTimeMember = value; - CheckedUtil.putWithoutChecking(super._map, "hourTime", value.data()); - } - - public static AbsoluteTimeRange.EndTime.ProjectionMask createMask() { - return new AbsoluteTimeRange.EndTime.ProjectionMask(); - } - - @Override - public AbsoluteTimeRange.EndTime clone() - throws CloneNotSupportedException - { - AbsoluteTimeRange.EndTime __clone = ((AbsoluteTimeRange.EndTime) super.clone()); - __clone.__changeListener = new AbsoluteTimeRange.EndTime.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public AbsoluteTimeRange.EndTime copy() - throws CloneNotSupportedException - { - AbsoluteTimeRange.EndTime __copy = ((AbsoluteTimeRange.EndTime) super.copy()); - __copy._dateMember = null; - __copy._hourTimeMember = null; - __copy.__changeListener = new AbsoluteTimeRange.EndTime.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final AbsoluteTimeRange.EndTime __objectRef; - - private ChangeListener(AbsoluteTimeRange.EndTime reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "date": - __objectRef._dateMember = null; - break; - case "hourTime": - __objectRef._hourTimeMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.config.join.Date.Fields Date() { - return new com.linkedin.feathr.config.join.Date.Fields(getPathComponents(), "date"); - } - - public com.linkedin.feathr.config.join.HourTime.Fields HourTime() { - return new com.linkedin.feathr.config.join.HourTime.Fields(getPathComponents(), "hourTime"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.Date.ProjectionMask _DateMask; - private com.linkedin.feathr.config.join.HourTime.ProjectionMask _HourTimeMask; - - ProjectionMask() { - super(3); - } - - public AbsoluteTimeRange.EndTime.ProjectionMask withDate(Function nestedMask) { - _DateMask = nestedMask.apply(((_DateMask == null)?com.linkedin.feathr.config.join.Date.createMask():_DateMask)); - getDataMap().put("date", _DateMask.getDataMap()); - return this; - } - - public AbsoluteTimeRange.EndTime.ProjectionMask withHourTime(Function nestedMask) { - _HourTimeMask = nestedMask.apply(((_HourTimeMask == null)?com.linkedin.feathr.config.join.HourTime.createMask():_HourTimeMask)); - getDataMap().put("hourTime", _HourTimeMask.getDataMap()); - return this; - } - - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * start time of the date range, in daily or hourly format with the start date included in the range. - * - */ - public com.linkedin.feathr.config.join.AbsoluteTimeRange.StartTime.Fields startTime() { - return new com.linkedin.feathr.config.join.AbsoluteTimeRange.StartTime.Fields(getPathComponents(), "startTime"); - } - - /** - * end date of the date range, in daily or hourly format with the end date included in the range. - * - */ - public com.linkedin.feathr.config.join.AbsoluteTimeRange.EndTime.Fields endTime() { - return new com.linkedin.feathr.config.join.AbsoluteTimeRange.EndTime.Fields(getPathComponents(), "endTime"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.AbsoluteTimeRange.StartTime.ProjectionMask _startTimeMask; - private com.linkedin.feathr.config.join.AbsoluteTimeRange.EndTime.ProjectionMask _endTimeMask; - - ProjectionMask() { - super(3); - } - - /** - * start time of the date range, in daily or hourly format with the start date included in the range. - * - */ - public AbsoluteTimeRange.ProjectionMask withStartTime(Function nestedMask) { - _startTimeMask = nestedMask.apply(((_startTimeMask == null)?AbsoluteTimeRange.StartTime.createMask():_startTimeMask)); - getDataMap().put("startTime", _startTimeMask.getDataMap()); - return this; - } - - /** - * start time of the date range, in daily or hourly format with the start date included in the range. - * - */ - public AbsoluteTimeRange.ProjectionMask withStartTime() { - _startTimeMask = null; - getDataMap().put("startTime", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * end date of the date range, in daily or hourly format with the end date included in the range. - * - */ - public AbsoluteTimeRange.ProjectionMask withEndTime(Function nestedMask) { - _endTimeMask = nestedMask.apply(((_endTimeMask == null)?AbsoluteTimeRange.EndTime.createMask():_endTimeMask)); - getDataMap().put("endTime", _endTimeMask.getDataMap()); - return this; - } - - /** - * end date of the date range, in daily or hourly format with the end date included in the range. - * - */ - public AbsoluteTimeRange.ProjectionMask withEndTime() { - _endTimeMask = null; - getDataMap().put("endTime", MaskMap.POSITIVE_MASK); - return this; - } - - } - - @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\AbsoluteTimeRange.pdl.") - public static class StartTime - extends UnionTemplate - { - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[date:{namespace com.linkedin.feathr.config.join/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}}hourTime:{namespace com.linkedin.feathr.config.join/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}}]", SchemaFormatType.PDL)); - private com.linkedin.feathr.config.join.Date _dateMember = null; - private com.linkedin.feathr.config.join.HourTime _hourTimeMember = null; - private AbsoluteTimeRange.StartTime.ChangeListener __changeListener = new AbsoluteTimeRange.StartTime.ChangeListener(this); - private final static DataSchema MEMBER_Date = SCHEMA.getTypeByMemberKey("date"); - private final static DataSchema MEMBER_HourTime = SCHEMA.getTypeByMemberKey("hourTime"); - - public StartTime() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public StartTime(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static AbsoluteTimeRange.StartTime createWithDate(com.linkedin.feathr.config.join.Date value) { - AbsoluteTimeRange.StartTime newUnion = new AbsoluteTimeRange.StartTime(); - newUnion.setDate(value); - return newUnion; - } - - public boolean isDate() { - return memberIs("date"); - } - - public com.linkedin.feathr.config.join.Date getDate() { - checkNotNull(); - if (_dateMember!= null) { - return _dateMember; - } - Object __rawValue = super._map.get("date"); - _dateMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.Date(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _dateMember; - } - - public void setDate(com.linkedin.feathr.config.join.Date value) { - checkNotNull(); - super._map.clear(); - _dateMember = value; - CheckedUtil.putWithoutChecking(super._map, "date", value.data()); - } - - public static AbsoluteTimeRange.StartTime createWithHourTime(com.linkedin.feathr.config.join.HourTime value) { - AbsoluteTimeRange.StartTime newUnion = new AbsoluteTimeRange.StartTime(); - newUnion.setHourTime(value); - return newUnion; - } - - public boolean isHourTime() { - return memberIs("hourTime"); - } - - public com.linkedin.feathr.config.join.HourTime getHourTime() { - checkNotNull(); - if (_hourTimeMember!= null) { - return _hourTimeMember; - } - Object __rawValue = super._map.get("hourTime"); - _hourTimeMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.HourTime(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _hourTimeMember; - } - - public void setHourTime(com.linkedin.feathr.config.join.HourTime value) { - checkNotNull(); - super._map.clear(); - _hourTimeMember = value; - CheckedUtil.putWithoutChecking(super._map, "hourTime", value.data()); - } - - public static AbsoluteTimeRange.StartTime.ProjectionMask createMask() { - return new AbsoluteTimeRange.StartTime.ProjectionMask(); - } - - @Override - public AbsoluteTimeRange.StartTime clone() - throws CloneNotSupportedException - { - AbsoluteTimeRange.StartTime __clone = ((AbsoluteTimeRange.StartTime) super.clone()); - __clone.__changeListener = new AbsoluteTimeRange.StartTime.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public AbsoluteTimeRange.StartTime copy() - throws CloneNotSupportedException - { - AbsoluteTimeRange.StartTime __copy = ((AbsoluteTimeRange.StartTime) super.copy()); - __copy._dateMember = null; - __copy._hourTimeMember = null; - __copy.__changeListener = new AbsoluteTimeRange.StartTime.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final AbsoluteTimeRange.StartTime __objectRef; - - private ChangeListener(AbsoluteTimeRange.StartTime reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "date": - __objectRef._dateMember = null; - break; - case "hourTime": - __objectRef._hourTimeMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.config.join.Date.Fields Date() { - return new com.linkedin.feathr.config.join.Date.Fields(getPathComponents(), "date"); - } - - public com.linkedin.feathr.config.join.HourTime.Fields HourTime() { - return new com.linkedin.feathr.config.join.HourTime.Fields(getPathComponents(), "hourTime"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.Date.ProjectionMask _DateMask; - private com.linkedin.feathr.config.join.HourTime.ProjectionMask _HourTimeMask; - - ProjectionMask() { - super(3); - } - - public AbsoluteTimeRange.StartTime.ProjectionMask withDate(Function nestedMask) { - _DateMask = nestedMask.apply(((_DateMask == null)?com.linkedin.feathr.config.join.Date.createMask():_DateMask)); - getDataMap().put("date", _DateMask.getDataMap()); - return this; - } - - public AbsoluteTimeRange.StartTime.ProjectionMask withHourTime(Function nestedMask) { - _HourTimeMask = nestedMask.apply(((_HourTimeMask == null)?com.linkedin.feathr.config.join.HourTime.createMask():_HourTimeMask)); - getDataMap().put("hourTime", _HourTimeMask.getDataMap()); - return this; - } - - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Date.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Date.java deleted file mode 100644 index c27cfb272..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Date.java +++ /dev/null @@ -1,575 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * Represents a date in a calendar year including day, year and month - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\Date.pdl.") -public class Date - extends RecordTemplate -{ - - private final static Date.Fields _fields = new Date.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}", SchemaFormatType.PDL)); - private Integer _dayField = null; - private Integer _monthField = null; - private Integer _yearField = null; - private Date.ChangeListener __changeListener = new Date.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Day = SCHEMA.getField("day"); - private final static RecordDataSchema.Field FIELD_Month = SCHEMA.getField("month"); - private final static RecordDataSchema.Field FIELD_Year = SCHEMA.getField("year"); - - public Date() { - super(new DataMap(4, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public Date(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static Date.Fields fields() { - return _fields; - } - - public static Date.ProjectionMask createMask() { - return new Date.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for day - * - * @see Date.Fields#day - */ - public boolean hasDay() { - if (_dayField!= null) { - return true; - } - return super._map.containsKey("day"); - } - - /** - * Remover for day - * - * @see Date.Fields#day - */ - public void removeDay() { - super._map.remove("day"); - } - - /** - * Getter for day - * - * @see Date.Fields#day - */ - public Integer getDay(GetMode mode) { - switch (mode) { - case STRICT: - return getDay(); - case DEFAULT: - case NULL: - if (_dayField!= null) { - return _dayField; - } else { - Object __rawValue = super._map.get("day"); - _dayField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _dayField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for day - * - * @return - * Required field. Could be null for partial record. - * @see Date.Fields#day - */ - @Nonnull - public Integer getDay() { - if (_dayField!= null) { - return _dayField; - } else { - Object __rawValue = super._map.get("day"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("day"); - } - _dayField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _dayField; - } - } - - /** - * Setter for day - * - * @see Date.Fields#day - */ - public Date setDay(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setDay(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field day of com.linkedin.feathr.config.join.Date"); - } else { - CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); - _dayField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeDay(); - } else { - CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); - _dayField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); - _dayField = value; - } - break; - } - return this; - } - - /** - * Setter for day - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Date.Fields#day - */ - public Date setDay( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field day of com.linkedin.feathr.config.join.Date to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); - _dayField = value; - } - return this; - } - - /** - * Setter for day - * - * @see Date.Fields#day - */ - public Date setDay(int value) { - CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); - _dayField = value; - return this; - } - - /** - * Existence checker for month - * - * @see Date.Fields#month - */ - public boolean hasMonth() { - if (_monthField!= null) { - return true; - } - return super._map.containsKey("month"); - } - - /** - * Remover for month - * - * @see Date.Fields#month - */ - public void removeMonth() { - super._map.remove("month"); - } - - /** - * Getter for month - * - * @see Date.Fields#month - */ - public Integer getMonth(GetMode mode) { - switch (mode) { - case STRICT: - return getMonth(); - case DEFAULT: - case NULL: - if (_monthField!= null) { - return _monthField; - } else { - Object __rawValue = super._map.get("month"); - _monthField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _monthField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for month - * - * @return - * Required field. Could be null for partial record. - * @see Date.Fields#month - */ - @Nonnull - public Integer getMonth() { - if (_monthField!= null) { - return _monthField; - } else { - Object __rawValue = super._map.get("month"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("month"); - } - _monthField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _monthField; - } - } - - /** - * Setter for month - * - * @see Date.Fields#month - */ - public Date setMonth(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setMonth(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field month of com.linkedin.feathr.config.join.Date"); - } else { - CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); - _monthField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeMonth(); - } else { - CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); - _monthField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); - _monthField = value; - } - break; - } - return this; - } - - /** - * Setter for month - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Date.Fields#month - */ - public Date setMonth( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field month of com.linkedin.feathr.config.join.Date to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); - _monthField = value; - } - return this; - } - - /** - * Setter for month - * - * @see Date.Fields#month - */ - public Date setMonth(int value) { - CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); - _monthField = value; - return this; - } - - /** - * Existence checker for year - * - * @see Date.Fields#year - */ - public boolean hasYear() { - if (_yearField!= null) { - return true; - } - return super._map.containsKey("year"); - } - - /** - * Remover for year - * - * @see Date.Fields#year - */ - public void removeYear() { - super._map.remove("year"); - } - - /** - * Getter for year - * - * @see Date.Fields#year - */ - public Integer getYear(GetMode mode) { - switch (mode) { - case STRICT: - return getYear(); - case DEFAULT: - case NULL: - if (_yearField!= null) { - return _yearField; - } else { - Object __rawValue = super._map.get("year"); - _yearField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _yearField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for year - * - * @return - * Required field. Could be null for partial record. - * @see Date.Fields#year - */ - @Nonnull - public Integer getYear() { - if (_yearField!= null) { - return _yearField; - } else { - Object __rawValue = super._map.get("year"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("year"); - } - _yearField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _yearField; - } - } - - /** - * Setter for year - * - * @see Date.Fields#year - */ - public Date setYear(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setYear(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field year of com.linkedin.feathr.config.join.Date"); - } else { - CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); - _yearField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeYear(); - } else { - CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); - _yearField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); - _yearField = value; - } - break; - } - return this; - } - - /** - * Setter for year - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Date.Fields#year - */ - public Date setYear( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field year of com.linkedin.feathr.config.join.Date to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); - _yearField = value; - } - return this; - } - - /** - * Setter for year - * - * @see Date.Fields#year - */ - public Date setYear(int value) { - CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); - _yearField = value; - return this; - } - - @Override - public Date clone() - throws CloneNotSupportedException - { - Date __clone = ((Date) super.clone()); - __clone.__changeListener = new Date.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public Date copy() - throws CloneNotSupportedException - { - Date __copy = ((Date) super.copy()); - __copy._monthField = null; - __copy._yearField = null; - __copy._dayField = null; - __copy.__changeListener = new Date.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final Date __objectRef; - - private ChangeListener(Date reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "month": - __objectRef._monthField = null; - break; - case "year": - __objectRef._yearField = null; - break; - case "day": - __objectRef._dayField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * day - * - */ - public PathSpec day() { - return new PathSpec(getPathComponents(), "day"); - } - - /** - * month - * - */ - public PathSpec month() { - return new PathSpec(getPathComponents(), "month"); - } - - /** - * year - * - */ - public PathSpec year() { - return new PathSpec(getPathComponents(), "year"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(4); - } - - /** - * day - * - */ - public Date.ProjectionMask withDay() { - getDataMap().put("day", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * month - * - */ - public Date.ProjectionMask withMonth() { - getDataMap().put("month", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * year - * - */ - public Date.ProjectionMask withYear() { - getDataMap().put("year", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/FrameFeatureJoinConfig.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/FrameFeatureJoinConfig.java deleted file mode 100644 index 6400da9df..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/FrameFeatureJoinConfig.java +++ /dev/null @@ -1,520 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * The join config consists of 2 parts, settings and features section. - * Settings is related to the general settings corresponding to joining the input data set with the - * features, currently there are time related settings, but this can be extended to other settings as well. - * Features to be joined are described by list of Keys and featureName and featureAlias. - * Features in the feature list should be joined to the user's input data. - * matching the key in the input data. - * For example, - * key is ["key1"] and join feature1 and feature2 with input data - * settings: { // optional field - * inputDataTimeSettings: { - * absoluteTimeRange: { - * startTime: Date(year=2020, month=4, day=28) - * endTime: Date(year=2020, month=5, day=5) - * } - * } - * joinTimeSettings: { - * timestampColumn: { - * def: timestamp - * format: yyyy-MM-dd - * } - * simulateTimeDelay: 5d - * } - * } - * features=[ - * JoiningFeature{ - * keys: ["key1"] - * frameFeatureName: "feature1" - * AbsoluteDateRange(startDate: Date(year=2020, month=5, day=1), - * endTime: Date(year=2020, month=5, day=5)) - * }, JoiningFeature{ - * keys: ["key1"] - * frameFeatureName: "feature2" - * overrideTimeDelay: 5d - * }, JoiningFeature{ - * keys: ["key1"] - * frameFeatureName: "feature3" - * RelativeDateRange(numDays: 5, - * offset: 3) - * }, JoiningFeature{ - * keys: ["key1"] - * frameFeatureName: "feature4" - * } - * ] - * - * Here, the keys are corresponding to column names in the input FeaturizedDataset, which will be used - * to join the feature source. Feature name is canonical feathr feature names. - * Each feature can also have a set of optional time-related parameters. These parameter override the ones provided in - * the settings section and are applicable only to the particular feature. - * Feature join config operation. - * - * All these PDLs are moved to feathr MP:- https://rb.corp.linkedin.com/r/2356512/ - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\FrameFeatureJoinConfig.pdl.") -public class FrameFeatureJoinConfig - extends RecordTemplate -{ - - private final static FrameFeatureJoinConfig.Fields _fields = new FrameFeatureJoinConfig.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The join config consists of 2 parts, settings and features section.\r\nSettings is related to the general settings corresponding to joining the input data set with the\r\nfeatures, currently there are time related settings, but this can be extended to other settings as well.\r\nFeatures to be joined are described by list of Keys and featureName and featureAlias.\r\nFeatures in the feature list should be joined to the user's input data.\r\nmatching the key in the input data.\r\nFor example,\r\nkey is [\"key1\"] and join feature1 and feature2 with input data\r\n settings: { // optional field\r\n inputDataTimeSettings: {\r\n absoluteTimeRange: {\r\n startTime: Date(year=2020, month=4, day=28)\r\n endTime: Date(year=2020, month=5, day=5)\r\n }\r\n }\r\n joinTimeSettings: {\r\n timestampColumn: {\r\n def: timestamp\r\n format: yyyy-MM-dd\r\n }\r\n simulateTimeDelay: 5d\r\n }\r\n }\r\n features=[\r\n JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature1\"\r\n AbsoluteDateRange(startDate: Date(year=2020, month=5, day=1),\r\n endTime: Date(year=2020, month=5, day=5))\r\n }, JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature2\"\r\n overrideTimeDelay: 5d\r\n }, JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature3\"\r\n RelativeDateRange(numDays: 5,\r\n offset: 3)\r\n }, JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature4\"\r\n }\r\n ]\r\n\r\nHere, the keys are corresponding to column names in the input FeaturizedDataset, which will be used\r\nto join the feature source. Feature name is canonical feathr feature names.\r\nEach feature can also have a set of optional time-related parameters. These parameter override the ones provided in\r\nthe settings section and are applicable only to the particular feature.\r\nFeature join config operation.\r\n\r\nAll these PDLs are moved to feathr MP:- https://rb.corp.linkedin.com/r/2356512/*/record FrameFeatureJoinConfig{/**settings required for joining input featurized dataset with the feature data.*/settings:optional/**The settings section contains all the config parameters required for the joining of input dataset with the\r\nfeature data. As of now, we have only time related parameters, but in future this can be expanded.\r\nThis section has configs related to:-\r\na. How do I load the input dataset if it is time sensitive?\r\nb. How do I specify the join parameters for input dataset?\r\nFor more details - https://docs.google.com/document/d/1C6u2CKWSmOmHDQEL8Ovm5V5ZZFKhC_HdxVxU9D1F9lg/edit#\r\nsettings: {\r\n inputDataTimeSettings: {\r\n absoluteTimeRange: {\r\n startTime: 20200809\r\n endTime: 20200810\r\n timeFormat: yyyyMMdd\r\n }\r\n }\r\n joinTimeSettings: {\r\n useLatestFeatureData: true\r\n }\r\n}*/record Settings{/**Config parameters related to loading of the time sensitive input data. Contains parameters related to restricting the\r\nsize of the input data with respect to the timestamp column.*/inputDataTimeSettings:optional/**The data time settings pertaining to how much of the input dataset is to be loaded from the timestamp column. This is a way in which\r\nthe input data can be restricted to allow only a fixed interval of dates to be joined with the feature data. This restriction\r\nwill apply on the timestamp column of the input data.\r\ninputDataTimeSettings: {\r\n absoluteTimeRange: {\r\n startTime: Date(year=2020, month=8, day=8)\r\n endTime: Date(year=2020, month=8, day=10)\r\n }\r\n (or)\r\n relativeTimeRange: {\r\n offset: TimeOffset(length=1, unit=\"DAY\")\r\n window: TimeWindow(length=1, unit=\"DAY\")\r\n }\r\n}*/record InputDataTimeSettings{/**Union of [[AbsoluteTimeRange]] and [[RelativeTimeRange]].\r\nIt indicates the range of input data which is to be loaded. This field generally refers to how much of the input\r\ndata should be restricted using the time in the timestamp column.\r\n\r\nFor example,\r\na. startDate: \"20200522\", endDate: \"20200525\" implies this feature should be joined with the input data starting from\r\n22nd May 2020 to 25th May, 2020 with both dates included.\r\nWe only support yyyyMMdd format for this. In future, if there is a request, we can\r\nadd support for other date time formats as well.\r\n\r\nb. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020\r\ntill 11/04/2020 willl be joined.*/timeRange:union[absoluteTimeRange:/**The absolute time range with start and end time being required fields.\r\nIt accepts a start time and an end time which should be specifiied using the [[Date.pdl]] or the [[HourTime.pdl]] class.\r\nThis model can be used to represent time range in daily or hourly interval.\r\nabsoluteTimeRange: {\r\n startTime: TimeHour(day=1, month=1, year=2020, hour=13)\r\n endTime: TimeHour(day=3, month=1, year=2020, hour=2)\r\n }\r\n(or)\r\nabsoluteTimeRange: {\r\n startTime: Date(day=1, month=1, year=2020)\r\n endTime: Date(day=3, month=1, year=2020)\r\n }\r\nendTime and startTime should always have the same granularity, ie - Daily or Hourly.\r\nendTme > startTime*/record AbsoluteTimeRange{/**start time of the date range, in daily or hourly format with the start date included in the range.*/startTime:union[date:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}hourTime:/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}]/**end date of the date range, in daily or hourly format with the end date included in the range.*/endTime:union[date:Date,hourTime:HourTime]}relativeTimeRange:/**The time range represented relative to the current timestamp. It uses the current system time as the reference and can be used to\r\nexpress a range of times with respect to the current time.\r\nExample, - If current time is 01/01/2020, window is 3 days, and offset is 1 day (unit can be day or hour).\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"DAY\"), offset=TimeOffset(length=1, unit=\"Day\"))\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"HOUR\"))*/record RelativeTimeRange{/**Window is the number of time units from the reference time units to look back to obtain the timeRange.\r\nFor example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020\r\ntill 11/05/2020 (both days included).\r\nwindow >= 1 TimeUnit*/window:/**Represents a length of time along with the corresponding time unit (DAY, HOUR).*/record TimeWindow{/**Amount of the duration in TimeUnits. Can be greater or equal to 1.*/@validate.positive,length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}/**Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time.\r\nFor example, if dateOffset is 4, and window is 2 days, then reference time\r\nwill be 4 days ago from today.\r\nExample - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020.\r\nThis will always take the window's timeUnits.*/@validate.integerRange.min=0,offset:long=0}]}/**This contains all the parameters required to join the time sensitive input data with the feature data.*/joinTimeSettings:optional/**JoinTimeSettings contains all the parameters required to join the time sensitive input data with the feature data.\r\nThe input data can be time sensitive in two ways:-\r\na. Have a timestamp column\r\nb. Always join with the latest available feature data. In this case, we do not require a timestamp column.\r\nc. The file path is time-partition and the path time is used for the join\r\n(Todo - Add useTimePartitionPattern field in this section)\r\nIn this section, the user needs to let feathr know which of the above properties is to be used for the join.*/typeref JoinTimeSettings=union[useLatestJoinTimeSettings:/**Settings needed when the input data is to be joined with the latest available feature data.\r\njoinTimeSettings: {\r\n useLatestFeatureData: true\r\n}*/record UseLatestJoinTimeSettings{/**Boolean value, if set to true, indicates that the latest available feature data is to be used for joining.\r\nWhen useLatestFeatureData is set, there should be no other time-based parameters.*/useLatestFeatureData:boolean=true}timestampColJoinTimeSettings:/**Settings needed when the input data has a timestamp which should be used for the join.\r\njoinTimeSettings: {\r\n timestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }\r\n simulateTimeDelay: 1d\r\n}*/record TimestampColJoinTimeSettings{/**The timestamp column name and timeformat which should be used for joining with the feature data.\r\nRefer to [[TimestampColumn]].\r\nExample, TimestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }*/timestampColumn:/**Timestamp column of the input featureiized dataset, which is to be used for the join.\r\ntimestampColumn: {\r\n def: timestamp\r\n format: yyyyMMdd\r\n }*/record TimestampColumn{/**The definiton of the timestamp column, which can be a sql expression involving the timestamp column\r\nor just the column name\r\nExample:- definition: timestamp, timestamp + 10000000.*/definition:union[columnName:string,sparkSqlExpression:/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}]/**Format of the timestamp column. Must confer to java's timestampFormatter or can be\r\nepoch or epoch_millis.\r\nExample:- epoch, epoch_millis, yyyy/MM/dd*/format:/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string}/**An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted\r\nfrom the input data timestamp while joining with the feature data.\r\nWe do support negative time delays.*/simulateTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:TimeUnit}}]}/**Array of joining features.\r\n\r\nValidation rules:\r\n- The array must be non-empty.*/features:array[/**JoiningFeature is the feature section of the join config. This section consists of information pertaining to a feature\r\nwhich is to be joined:-\r\na. The join keys of the input data, with which this feature is to be joined.\r\nb. name of the feature\r\nc. optional timeRange of the input data which is to be joined with this feature.\r\nd. optional overrideTimeDelay if this feature needs a different simulate time delay other than the one mentioned.\r\n\r\nThis is a required section of the the join config.\r\nExample,\r\n a. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature1\"\r\n AbsoluteDateRange(startDate: Date(year=2020, month=5, day=5),\r\n endDate: Date(year=2020, month=5, day=7))\r\n }\r\n b. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature2\"\r\n overrideTimeDelay: TimeDelay(length=1, unit=\"DAY\")\r\n }\r\n c. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature3\"\r\n RelativeDateRange(numDays: 5,\r\n offset: 3)\r\n }*/record JoiningFeature{/**Keys to join input with feature source, the field name of the key in the input featuized dataset.*/keys:array[string]/**Feature name as defined in feathr's feature definition configuration.\r\n\r\nCurrently the column in the output FDS that holds this feature will have the same name as feature name.\r\nIf multiple joined features have the same name and no alias is defined for them, feathr will prepend the keys to the feature name.\r\n\r\nIn the future, if \"featureAlias\" is not set, the column in the output FDS that holds this feature will have the same name as feature name.\r\nIf multiple joined features have the same name and no alias is defined for them, the join operation will fail\r\n(to avoid produciing two columns in the output FDS with the same name).*/frameFeatureName:string/**The development of this is in progress. This is not in use for now.\r\n\r\nThe name to be used for the column in the output FDS that contains the values from this joined feature.\r\nIf not set, the name of the feature (frameFeatureName) will be used for the output column.\r\nFor example, if the user request joining a feature named \"careers_job_listTime\" and provides no alias,\r\nthe output FDS will contain a column called \"careers_job_listTime\". However, if the user sets \"featureAlias\" to \"list_time\",\r\nthe column will be named \"list_time\".\r\n\r\nfeature alias can be useful for in a few cases:\r\n - If the user prefers to use a name different than the feathr name in their model,\r\nthey can use an alias to control the name of the column in the output FDS.\r\n - Sometimes, the training datas needs to have two features that are from the same feathr feature.\r\nFor example, if we are modeing the problem of the probability of a member A (viewer) seeing the profile of member B\r\n(viewee) and we want to use the skills of both viewer and viewee as features, we need to join feathr feature\r\n\"member_skills\" of member A with feathr feature \"member_skills\" of member B. That is, the two features are the same\r\nfeature but for different entiity ids). The default behavior of join is to name the output column name using the feathr\r\nfeature name, but in a case like the above case, that would result in two columns with the same name,\r\nwhich is not valid for FDS. In these cases, the user has to provide an alias for at least one of these joined features.\r\nFor example, the user can use featureAliases such as \"viewer_skills\" and \"viewee_skills\".\r\nIn these cases, featureAliases becomes mandatory.*/featureAlias:optional string/**dateRange is used in Time-based joins, which refers to the situation when one or multiple days of input data needs\r\nto be used for training.\r\nOne of the common use cases where this is used, is in training with some time-insensitive features, or\r\ntraining pipeline that always use the full day data, one day before running (since there is only partial data for today).\r\nThe time for the input featurized dataset can be set using this field.\r\nHourly data is not allowed in this case.\r\n\r\nFor example,\r\na. startDate: \"20200522\", endDate: \"20200525\" implies this feature should be joined with the input data starting from\r\n22nd May 2020 to 25th May, 2020 with both dates included.\r\nWe only support yyyyMMdd format for this. In future, if there is a request, we can\r\nadd support for other date time formats as well.\r\n\r\nb. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020\r\ntill 11/04/2020 willl be joined.\r\n\r\nP.S - This is different from the timeRange used in settings as the settings startTime is applicable for the entire input data,\r\nwhile this a feature level setting. Also, we do not support hourly time here.*/dateRange:optional union[absoluteDateRange:/**The absolute date range with start and end date being required fields.\r\nIt accepts a start date and an end date which should be specifiied using the [[Date.pdl]] class.\r\nabsoluteDateRange: {\r\n startDate: Date(day=1, month=1, year=2020)\r\n endDate: Date(day=3, month=1, year=2020)\r\n }\r\n In this case, the endDate > startDate.*/record AbsoluteDateRange{/**start date of the date range, with the start date included in the range.*/startDate:Date/**end date of the date range, with the end date included in the range.*/endDate:Date}relativeDateRange:/**The date range represented relative to the current date. It uses the current system date as the reference and can be used to\r\nexpress a range of dates with respect to the current date.\r\nExample, - If current date is 01/01/2020, window is 3, and offset 1 (unit is number of days)\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nIf dateOffset is not specified, it defaults to 0.\r\nrelativeDateRange: RelativeDateRange(numDays=2, dateOffset=1)\r\nrelativeDateRange: RelativeDateRange(numDays=5)*/record RelativeDateRange{/**Represents a length of time.\r\nnumDays is the window from the reference date to look back to obtain a dateRange.\r\nFor example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020\r\ntill 11/05/2020.*/@validate.positive={}numDays:long/**Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date\r\nwill be 4 days ago from today.*/dateOffset:long=0}]/**The override time delay parameter which will override the global simulate time delay specified in the settings section for\r\nthe particular feature.\r\nThis parameter is only applicable when the simulate time delay is set in the settings section\r\nFor example, let us say the global simulate delay was 5d, and the overrideTimeDelay is set to 3d.\r\nThen, for this specificc feature, a simulate delay of 3d will be applied.*/overrideTimeDelay:optional TimeOffset}]}", SchemaFormatType.PDL)); - private Settings _settingsField = null; - private JoiningFeatureArray _featuresField = null; - private FrameFeatureJoinConfig.ChangeListener __changeListener = new FrameFeatureJoinConfig.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Settings = SCHEMA.getField("settings"); - private final static RecordDataSchema.Field FIELD_Features = SCHEMA.getField("features"); - - public FrameFeatureJoinConfig() { - super(new DataMap(3, 0.75F), SCHEMA, 3); - addChangeListener(__changeListener); - } - - public FrameFeatureJoinConfig(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static FrameFeatureJoinConfig.Fields fields() { - return _fields; - } - - public static FrameFeatureJoinConfig.ProjectionMask createMask() { - return new FrameFeatureJoinConfig.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for settings - * - * @see FrameFeatureJoinConfig.Fields#settings - */ - public boolean hasSettings() { - if (_settingsField!= null) { - return true; - } - return super._map.containsKey("settings"); - } - - /** - * Remover for settings - * - * @see FrameFeatureJoinConfig.Fields#settings - */ - public void removeSettings() { - super._map.remove("settings"); - } - - /** - * Getter for settings - * - * @see FrameFeatureJoinConfig.Fields#settings - */ - public Settings getSettings(GetMode mode) { - return getSettings(); - } - - /** - * Getter for settings - * - * @return - * Optional field. Always check for null. - * @see FrameFeatureJoinConfig.Fields#settings - */ - @Nullable - public Settings getSettings() { - if (_settingsField!= null) { - return _settingsField; - } else { - Object __rawValue = super._map.get("settings"); - _settingsField = ((__rawValue == null)?null:new Settings(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _settingsField; - } - } - - /** - * Setter for settings - * - * @see FrameFeatureJoinConfig.Fields#settings - */ - public FrameFeatureJoinConfig setSettings(Settings value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setSettings(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeSettings(); - } else { - CheckedUtil.putWithoutChecking(super._map, "settings", value.data()); - _settingsField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "settings", value.data()); - _settingsField = value; - } - break; - } - return this; - } - - /** - * Setter for settings - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see FrameFeatureJoinConfig.Fields#settings - */ - public FrameFeatureJoinConfig setSettings( - @Nonnull - Settings value) { - if (value == null) { - throw new NullPointerException("Cannot set field settings of com.linkedin.feathr.config.join.FrameFeatureJoinConfig to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "settings", value.data()); - _settingsField = value; - } - return this; - } - - /** - * Existence checker for features - * - * @see FrameFeatureJoinConfig.Fields#features - */ - public boolean hasFeatures() { - if (_featuresField!= null) { - return true; - } - return super._map.containsKey("features"); - } - - /** - * Remover for features - * - * @see FrameFeatureJoinConfig.Fields#features - */ - public void removeFeatures() { - super._map.remove("features"); - } - - /** - * Getter for features - * - * @see FrameFeatureJoinConfig.Fields#features - */ - public JoiningFeatureArray getFeatures(GetMode mode) { - switch (mode) { - case STRICT: - return getFeatures(); - case DEFAULT: - case NULL: - if (_featuresField!= null) { - return _featuresField; - } else { - Object __rawValue = super._map.get("features"); - _featuresField = ((__rawValue == null)?null:new JoiningFeatureArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _featuresField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for features - * - * @return - * Required field. Could be null for partial record. - * @see FrameFeatureJoinConfig.Fields#features - */ - @Nonnull - public JoiningFeatureArray getFeatures() { - if (_featuresField!= null) { - return _featuresField; - } else { - Object __rawValue = super._map.get("features"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("features"); - } - _featuresField = ((__rawValue == null)?null:new JoiningFeatureArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _featuresField; - } - } - - /** - * Setter for features - * - * @see FrameFeatureJoinConfig.Fields#features - */ - public FrameFeatureJoinConfig setFeatures(JoiningFeatureArray value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFeatures(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field features of com.linkedin.feathr.config.join.FrameFeatureJoinConfig"); - } else { - CheckedUtil.putWithoutChecking(super._map, "features", value.data()); - _featuresField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeFeatures(); - } else { - CheckedUtil.putWithoutChecking(super._map, "features", value.data()); - _featuresField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "features", value.data()); - _featuresField = value; - } - break; - } - return this; - } - - /** - * Setter for features - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see FrameFeatureJoinConfig.Fields#features - */ - public FrameFeatureJoinConfig setFeatures( - @Nonnull - JoiningFeatureArray value) { - if (value == null) { - throw new NullPointerException("Cannot set field features of com.linkedin.feathr.config.join.FrameFeatureJoinConfig to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "features", value.data()); - _featuresField = value; - } - return this; - } - - @Override - public FrameFeatureJoinConfig clone() - throws CloneNotSupportedException - { - FrameFeatureJoinConfig __clone = ((FrameFeatureJoinConfig) super.clone()); - __clone.__changeListener = new FrameFeatureJoinConfig.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public FrameFeatureJoinConfig copy() - throws CloneNotSupportedException - { - FrameFeatureJoinConfig __copy = ((FrameFeatureJoinConfig) super.copy()); - __copy._settingsField = null; - __copy._featuresField = null; - __copy.__changeListener = new FrameFeatureJoinConfig.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final FrameFeatureJoinConfig __objectRef; - - private ChangeListener(FrameFeatureJoinConfig reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "settings": - __objectRef._settingsField = null; - break; - case "features": - __objectRef._featuresField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * settings required for joining input featurized dataset with the feature data. - * - */ - public com.linkedin.feathr.config.join.Settings.Fields settings() { - return new com.linkedin.feathr.config.join.Settings.Fields(getPathComponents(), "settings"); - } - - /** - * Array of joining features. - * - * Validation rules: - * - The array must be non-empty. - * - */ - public com.linkedin.feathr.config.join.JoiningFeatureArray.Fields features() { - return new com.linkedin.feathr.config.join.JoiningFeatureArray.Fields(getPathComponents(), "features"); - } - - /** - * Array of joining features. - * - * Validation rules: - * - The array must be non-empty. - * - */ - public PathSpec features(Integer start, Integer count) { - PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "features"); - if (start!= null) { - arrayPathSpec.setAttribute("start", start); - } - if (count!= null) { - arrayPathSpec.setAttribute("count", count); - } - return arrayPathSpec; - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.Settings.ProjectionMask _settingsMask; - private com.linkedin.feathr.config.join.JoiningFeatureArray.ProjectionMask _featuresMask; - - ProjectionMask() { - super(3); - } - - /** - * settings required for joining input featurized dataset with the feature data. - * - */ - public FrameFeatureJoinConfig.ProjectionMask withSettings(Function nestedMask) { - _settingsMask = nestedMask.apply(((_settingsMask == null)?Settings.createMask():_settingsMask)); - getDataMap().put("settings", _settingsMask.getDataMap()); - return this; - } - - /** - * settings required for joining input featurized dataset with the feature data. - * - */ - public FrameFeatureJoinConfig.ProjectionMask withSettings() { - _settingsMask = null; - getDataMap().put("settings", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Array of joining features. - * - * Validation rules: - * - The array must be non-empty. - * - */ - public FrameFeatureJoinConfig.ProjectionMask withFeatures(Function nestedMask) { - _featuresMask = nestedMask.apply(((_featuresMask == null)?JoiningFeatureArray.createMask():_featuresMask)); - getDataMap().put("features", _featuresMask.getDataMap()); - return this; - } - - /** - * Array of joining features. - * - * Validation rules: - * - The array must be non-empty. - * - */ - public FrameFeatureJoinConfig.ProjectionMask withFeatures() { - _featuresMask = null; - getDataMap().put("features", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Array of joining features. - * - * Validation rules: - * - The array must be non-empty. - * - */ - public FrameFeatureJoinConfig.ProjectionMask withFeatures(Function nestedMask, Integer start, Integer count) { - _featuresMask = nestedMask.apply(((_featuresMask == null)?JoiningFeatureArray.createMask():_featuresMask)); - getDataMap().put("features", _featuresMask.getDataMap()); - if (start!= null) { - getDataMap().getDataMap("features").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("features").put("$count", count); - } - return this; - } - - /** - * Array of joining features. - * - * Validation rules: - * - The array must be non-empty. - * - */ - public FrameFeatureJoinConfig.ProjectionMask withFeatures(Integer start, Integer count) { - _featuresMask = null; - getDataMap().put("features", new DataMap(3)); - if (start!= null) { - getDataMap().getDataMap("features").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("features").put("$count", count); - } - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/HourTime.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/HourTime.java deleted file mode 100644 index 7421350ae..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/HourTime.java +++ /dev/null @@ -1,727 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * Time with hourly granularity - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\HourTime.pdl.") -public class HourTime - extends RecordTemplate -{ - - private final static HourTime.Fields _fields = new HourTime.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}", SchemaFormatType.PDL)); - private Integer _dayField = null; - private Integer _monthField = null; - private Integer _yearField = null; - private Integer _hourField = null; - private HourTime.ChangeListener __changeListener = new HourTime.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Day = SCHEMA.getField("day"); - private final static RecordDataSchema.Field FIELD_Month = SCHEMA.getField("month"); - private final static RecordDataSchema.Field FIELD_Year = SCHEMA.getField("year"); - private final static RecordDataSchema.Field FIELD_Hour = SCHEMA.getField("hour"); - - public HourTime() { - super(new DataMap(6, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public HourTime(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static HourTime.Fields fields() { - return _fields; - } - - public static HourTime.ProjectionMask createMask() { - return new HourTime.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for day - * - * @see HourTime.Fields#day - */ - public boolean hasDay() { - if (_dayField!= null) { - return true; - } - return super._map.containsKey("day"); - } - - /** - * Remover for day - * - * @see HourTime.Fields#day - */ - public void removeDay() { - super._map.remove("day"); - } - - /** - * Getter for day - * - * @see HourTime.Fields#day - */ - public Integer getDay(GetMode mode) { - switch (mode) { - case STRICT: - return getDay(); - case DEFAULT: - case NULL: - if (_dayField!= null) { - return _dayField; - } else { - Object __rawValue = super._map.get("day"); - _dayField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _dayField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for day - * - * @return - * Required field. Could be null for partial record. - * @see HourTime.Fields#day - */ - @Nonnull - public Integer getDay() { - if (_dayField!= null) { - return _dayField; - } else { - Object __rawValue = super._map.get("day"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("day"); - } - _dayField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _dayField; - } - } - - /** - * Setter for day - * - * @see HourTime.Fields#day - */ - public HourTime setDay(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setDay(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field day of com.linkedin.feathr.config.join.HourTime"); - } else { - CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); - _dayField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeDay(); - } else { - CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); - _dayField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); - _dayField = value; - } - break; - } - return this; - } - - /** - * Setter for day - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see HourTime.Fields#day - */ - public HourTime setDay( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field day of com.linkedin.feathr.config.join.HourTime to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); - _dayField = value; - } - return this; - } - - /** - * Setter for day - * - * @see HourTime.Fields#day - */ - public HourTime setDay(int value) { - CheckedUtil.putWithoutChecking(super._map, "day", DataTemplateUtil.coerceIntInput(value)); - _dayField = value; - return this; - } - - /** - * Existence checker for month - * - * @see HourTime.Fields#month - */ - public boolean hasMonth() { - if (_monthField!= null) { - return true; - } - return super._map.containsKey("month"); - } - - /** - * Remover for month - * - * @see HourTime.Fields#month - */ - public void removeMonth() { - super._map.remove("month"); - } - - /** - * Getter for month - * - * @see HourTime.Fields#month - */ - public Integer getMonth(GetMode mode) { - switch (mode) { - case STRICT: - return getMonth(); - case DEFAULT: - case NULL: - if (_monthField!= null) { - return _monthField; - } else { - Object __rawValue = super._map.get("month"); - _monthField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _monthField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for month - * - * @return - * Required field. Could be null for partial record. - * @see HourTime.Fields#month - */ - @Nonnull - public Integer getMonth() { - if (_monthField!= null) { - return _monthField; - } else { - Object __rawValue = super._map.get("month"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("month"); - } - _monthField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _monthField; - } - } - - /** - * Setter for month - * - * @see HourTime.Fields#month - */ - public HourTime setMonth(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setMonth(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field month of com.linkedin.feathr.config.join.HourTime"); - } else { - CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); - _monthField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeMonth(); - } else { - CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); - _monthField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); - _monthField = value; - } - break; - } - return this; - } - - /** - * Setter for month - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see HourTime.Fields#month - */ - public HourTime setMonth( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field month of com.linkedin.feathr.config.join.HourTime to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); - _monthField = value; - } - return this; - } - - /** - * Setter for month - * - * @see HourTime.Fields#month - */ - public HourTime setMonth(int value) { - CheckedUtil.putWithoutChecking(super._map, "month", DataTemplateUtil.coerceIntInput(value)); - _monthField = value; - return this; - } - - /** - * Existence checker for year - * - * @see HourTime.Fields#year - */ - public boolean hasYear() { - if (_yearField!= null) { - return true; - } - return super._map.containsKey("year"); - } - - /** - * Remover for year - * - * @see HourTime.Fields#year - */ - public void removeYear() { - super._map.remove("year"); - } - - /** - * Getter for year - * - * @see HourTime.Fields#year - */ - public Integer getYear(GetMode mode) { - switch (mode) { - case STRICT: - return getYear(); - case DEFAULT: - case NULL: - if (_yearField!= null) { - return _yearField; - } else { - Object __rawValue = super._map.get("year"); - _yearField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _yearField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for year - * - * @return - * Required field. Could be null for partial record. - * @see HourTime.Fields#year - */ - @Nonnull - public Integer getYear() { - if (_yearField!= null) { - return _yearField; - } else { - Object __rawValue = super._map.get("year"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("year"); - } - _yearField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _yearField; - } - } - - /** - * Setter for year - * - * @see HourTime.Fields#year - */ - public HourTime setYear(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setYear(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field year of com.linkedin.feathr.config.join.HourTime"); - } else { - CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); - _yearField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeYear(); - } else { - CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); - _yearField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); - _yearField = value; - } - break; - } - return this; - } - - /** - * Setter for year - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see HourTime.Fields#year - */ - public HourTime setYear( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field year of com.linkedin.feathr.config.join.HourTime to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); - _yearField = value; - } - return this; - } - - /** - * Setter for year - * - * @see HourTime.Fields#year - */ - public HourTime setYear(int value) { - CheckedUtil.putWithoutChecking(super._map, "year", DataTemplateUtil.coerceIntInput(value)); - _yearField = value; - return this; - } - - /** - * Existence checker for hour - * - * @see HourTime.Fields#hour - */ - public boolean hasHour() { - if (_hourField!= null) { - return true; - } - return super._map.containsKey("hour"); - } - - /** - * Remover for hour - * - * @see HourTime.Fields#hour - */ - public void removeHour() { - super._map.remove("hour"); - } - - /** - * Getter for hour - * - * @see HourTime.Fields#hour - */ - public Integer getHour(GetMode mode) { - switch (mode) { - case STRICT: - return getHour(); - case DEFAULT: - case NULL: - if (_hourField!= null) { - return _hourField; - } else { - Object __rawValue = super._map.get("hour"); - _hourField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _hourField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for hour - * - * @return - * Required field. Could be null for partial record. - * @see HourTime.Fields#hour - */ - @Nonnull - public Integer getHour() { - if (_hourField!= null) { - return _hourField; - } else { - Object __rawValue = super._map.get("hour"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("hour"); - } - _hourField = DataTemplateUtil.coerceIntOutput(__rawValue); - return _hourField; - } - } - - /** - * Setter for hour - * - * @see HourTime.Fields#hour - */ - public HourTime setHour(Integer value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setHour(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field hour of com.linkedin.feathr.config.join.HourTime"); - } else { - CheckedUtil.putWithoutChecking(super._map, "hour", DataTemplateUtil.coerceIntInput(value)); - _hourField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeHour(); - } else { - CheckedUtil.putWithoutChecking(super._map, "hour", DataTemplateUtil.coerceIntInput(value)); - _hourField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "hour", DataTemplateUtil.coerceIntInput(value)); - _hourField = value; - } - break; - } - return this; - } - - /** - * Setter for hour - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see HourTime.Fields#hour - */ - public HourTime setHour( - @Nonnull - Integer value) { - if (value == null) { - throw new NullPointerException("Cannot set field hour of com.linkedin.feathr.config.join.HourTime to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "hour", DataTemplateUtil.coerceIntInput(value)); - _hourField = value; - } - return this; - } - - /** - * Setter for hour - * - * @see HourTime.Fields#hour - */ - public HourTime setHour(int value) { - CheckedUtil.putWithoutChecking(super._map, "hour", DataTemplateUtil.coerceIntInput(value)); - _hourField = value; - return this; - } - - @Override - public HourTime clone() - throws CloneNotSupportedException - { - HourTime __clone = ((HourTime) super.clone()); - __clone.__changeListener = new HourTime.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public HourTime copy() - throws CloneNotSupportedException - { - HourTime __copy = ((HourTime) super.copy()); - __copy._monthField = null; - __copy._hourField = null; - __copy._yearField = null; - __copy._dayField = null; - __copy.__changeListener = new HourTime.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final HourTime __objectRef; - - private ChangeListener(HourTime reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "month": - __objectRef._monthField = null; - break; - case "hour": - __objectRef._hourField = null; - break; - case "year": - __objectRef._yearField = null; - break; - case "day": - __objectRef._dayField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * day - * - */ - public PathSpec day() { - return new PathSpec(getPathComponents(), "day"); - } - - /** - * month - * - */ - public PathSpec month() { - return new PathSpec(getPathComponents(), "month"); - } - - /** - * year - * - */ - public PathSpec year() { - return new PathSpec(getPathComponents(), "year"); - } - - /** - * hour - * - */ - public PathSpec hour() { - return new PathSpec(getPathComponents(), "hour"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(6); - } - - /** - * day - * - */ - public HourTime.ProjectionMask withDay() { - getDataMap().put("day", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * month - * - */ - public HourTime.ProjectionMask withMonth() { - getDataMap().put("month", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * year - * - */ - public HourTime.ProjectionMask withYear() { - getDataMap().put("year", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * hour - * - */ - public HourTime.ProjectionMask withHour() { - getDataMap().put("hour", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/InputDataTimeSettings.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/InputDataTimeSettings.java deleted file mode 100644 index ec2c535cb..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/InputDataTimeSettings.java +++ /dev/null @@ -1,502 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.DataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.UnionDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; -import com.linkedin.data.template.UnionTemplate; - - -/** - * The data time settings pertaining to how much of the input dataset is to be loaded from the timestamp column. This is a way in which - * the input data can be restricted to allow only a fixed interval of dates to be joined with the feature data. This restriction - * will apply on the timestamp column of the input data. - * inputDataTimeSettings: { - * absoluteTimeRange: { - * startTime: Date(year=2020, month=8, day=8) - * endTime: Date(year=2020, month=8, day=10) - * } - * (or) - * relativeTimeRange: { - * offset: TimeOffset(length=1, unit="DAY") - * window: TimeWindow(length=1, unit="DAY") - * } - * } - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\InputDataTimeSettings.pdl.") -public class InputDataTimeSettings - extends RecordTemplate -{ - - private final static InputDataTimeSettings.Fields _fields = new InputDataTimeSettings.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The data time settings pertaining to how much of the input dataset is to be loaded from the timestamp column. This is a way in which\r\nthe input data can be restricted to allow only a fixed interval of dates to be joined with the feature data. This restriction\r\nwill apply on the timestamp column of the input data.\r\ninputDataTimeSettings: {\r\n absoluteTimeRange: {\r\n startTime: Date(year=2020, month=8, day=8)\r\n endTime: Date(year=2020, month=8, day=10)\r\n }\r\n (or)\r\n relativeTimeRange: {\r\n offset: TimeOffset(length=1, unit=\"DAY\")\r\n window: TimeWindow(length=1, unit=\"DAY\")\r\n }\r\n}*/record InputDataTimeSettings{/**Union of [[AbsoluteTimeRange]] and [[RelativeTimeRange]].\r\nIt indicates the range of input data which is to be loaded. This field generally refers to how much of the input\r\ndata should be restricted using the time in the timestamp column.\r\n\r\nFor example,\r\na. startDate: \"20200522\", endDate: \"20200525\" implies this feature should be joined with the input data starting from\r\n22nd May 2020 to 25th May, 2020 with both dates included.\r\nWe only support yyyyMMdd format for this. In future, if there is a request, we can\r\nadd support for other date time formats as well.\r\n\r\nb. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020\r\ntill 11/04/2020 willl be joined.*/timeRange:union[absoluteTimeRange:/**The absolute time range with start and end time being required fields.\r\nIt accepts a start time and an end time which should be specifiied using the [[Date.pdl]] or the [[HourTime.pdl]] class.\r\nThis model can be used to represent time range in daily or hourly interval.\r\nabsoluteTimeRange: {\r\n startTime: TimeHour(day=1, month=1, year=2020, hour=13)\r\n endTime: TimeHour(day=3, month=1, year=2020, hour=2)\r\n }\r\n(or)\r\nabsoluteTimeRange: {\r\n startTime: Date(day=1, month=1, year=2020)\r\n endTime: Date(day=3, month=1, year=2020)\r\n }\r\nendTime and startTime should always have the same granularity, ie - Daily or Hourly.\r\nendTme > startTime*/record AbsoluteTimeRange{/**start time of the date range, in daily or hourly format with the start date included in the range.*/startTime:union[date:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}hourTime:/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}]/**end date of the date range, in daily or hourly format with the end date included in the range.*/endTime:union[date:Date,hourTime:HourTime]}relativeTimeRange:/**The time range represented relative to the current timestamp. It uses the current system time as the reference and can be used to\r\nexpress a range of times with respect to the current time.\r\nExample, - If current time is 01/01/2020, window is 3 days, and offset is 1 day (unit can be day or hour).\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"DAY\"), offset=TimeOffset(length=1, unit=\"Day\"))\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"HOUR\"))*/record RelativeTimeRange{/**Window is the number of time units from the reference time units to look back to obtain the timeRange.\r\nFor example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020\r\ntill 11/05/2020 (both days included).\r\nwindow >= 1 TimeUnit*/window:/**Represents a length of time along with the corresponding time unit (DAY, HOUR).*/record TimeWindow{/**Amount of the duration in TimeUnits. Can be greater or equal to 1.*/@validate.positive,length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}/**Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time.\r\nFor example, if dateOffset is 4, and window is 2 days, then reference time\r\nwill be 4 days ago from today.\r\nExample - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020.\r\nThis will always take the window's timeUnits.*/@validate.integerRange.min=0,offset:long=0}]}", SchemaFormatType.PDL)); - private InputDataTimeSettings.TimeRange _timeRangeField = null; - private InputDataTimeSettings.ChangeListener __changeListener = new InputDataTimeSettings.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_TimeRange = SCHEMA.getField("timeRange"); - - public InputDataTimeSettings() { - super(new DataMap(2, 0.75F), SCHEMA, 2); - addChangeListener(__changeListener); - } - - public InputDataTimeSettings(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static InputDataTimeSettings.Fields fields() { - return _fields; - } - - public static InputDataTimeSettings.ProjectionMask createMask() { - return new InputDataTimeSettings.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for timeRange - * - * @see InputDataTimeSettings.Fields#timeRange - */ - public boolean hasTimeRange() { - if (_timeRangeField!= null) { - return true; - } - return super._map.containsKey("timeRange"); - } - - /** - * Remover for timeRange - * - * @see InputDataTimeSettings.Fields#timeRange - */ - public void removeTimeRange() { - super._map.remove("timeRange"); - } - - /** - * Getter for timeRange - * - * @see InputDataTimeSettings.Fields#timeRange - */ - public InputDataTimeSettings.TimeRange getTimeRange(GetMode mode) { - switch (mode) { - case STRICT: - return getTimeRange(); - case DEFAULT: - case NULL: - if (_timeRangeField!= null) { - return _timeRangeField; - } else { - Object __rawValue = super._map.get("timeRange"); - _timeRangeField = ((__rawValue == null)?null:new InputDataTimeSettings.TimeRange(__rawValue)); - return _timeRangeField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for timeRange - * - * @return - * Required field. Could be null for partial record. - * @see InputDataTimeSettings.Fields#timeRange - */ - @Nonnull - public InputDataTimeSettings.TimeRange getTimeRange() { - if (_timeRangeField!= null) { - return _timeRangeField; - } else { - Object __rawValue = super._map.get("timeRange"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("timeRange"); - } - _timeRangeField = ((__rawValue == null)?null:new InputDataTimeSettings.TimeRange(__rawValue)); - return _timeRangeField; - } - } - - /** - * Setter for timeRange - * - * @see InputDataTimeSettings.Fields#timeRange - */ - public InputDataTimeSettings setTimeRange(InputDataTimeSettings.TimeRange value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setTimeRange(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field timeRange of com.linkedin.feathr.config.join.InputDataTimeSettings"); - } else { - CheckedUtil.putWithoutChecking(super._map, "timeRange", value.data()); - _timeRangeField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeTimeRange(); - } else { - CheckedUtil.putWithoutChecking(super._map, "timeRange", value.data()); - _timeRangeField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "timeRange", value.data()); - _timeRangeField = value; - } - break; - } - return this; - } - - /** - * Setter for timeRange - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see InputDataTimeSettings.Fields#timeRange - */ - public InputDataTimeSettings setTimeRange( - @Nonnull - InputDataTimeSettings.TimeRange value) { - if (value == null) { - throw new NullPointerException("Cannot set field timeRange of com.linkedin.feathr.config.join.InputDataTimeSettings to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "timeRange", value.data()); - _timeRangeField = value; - } - return this; - } - - @Override - public InputDataTimeSettings clone() - throws CloneNotSupportedException - { - InputDataTimeSettings __clone = ((InputDataTimeSettings) super.clone()); - __clone.__changeListener = new InputDataTimeSettings.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public InputDataTimeSettings copy() - throws CloneNotSupportedException - { - InputDataTimeSettings __copy = ((InputDataTimeSettings) super.copy()); - __copy._timeRangeField = null; - __copy.__changeListener = new InputDataTimeSettings.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final InputDataTimeSettings __objectRef; - - private ChangeListener(InputDataTimeSettings reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "timeRange": - __objectRef._timeRangeField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Union of [[AbsoluteTimeRange]] and [[RelativeTimeRange]]. - * It indicates the range of input data which is to be loaded. This field generally refers to how much of the input - * data should be restricted using the time in the timestamp column. - * - * For example, - * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from - * 22nd May 2020 to 25th May, 2020 with both dates included. - * We only support yyyyMMdd format for this. In future, if there is a request, we can - * add support for other date time formats as well. - * - * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 - * till 11/04/2020 willl be joined. - * - */ - public com.linkedin.feathr.config.join.InputDataTimeSettings.TimeRange.Fields timeRange() { - return new com.linkedin.feathr.config.join.InputDataTimeSettings.TimeRange.Fields(getPathComponents(), "timeRange"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.InputDataTimeSettings.TimeRange.ProjectionMask _timeRangeMask; - - ProjectionMask() { - super(2); - } - - /** - * Union of [[AbsoluteTimeRange]] and [[RelativeTimeRange]]. - * It indicates the range of input data which is to be loaded. This field generally refers to how much of the input - * data should be restricted using the time in the timestamp column. - * - * For example, - * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from - * 22nd May 2020 to 25th May, 2020 with both dates included. - * We only support yyyyMMdd format for this. In future, if there is a request, we can - * add support for other date time formats as well. - * - * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 - * till 11/04/2020 willl be joined. - * - */ - public InputDataTimeSettings.ProjectionMask withTimeRange(Function nestedMask) { - _timeRangeMask = nestedMask.apply(((_timeRangeMask == null)?InputDataTimeSettings.TimeRange.createMask():_timeRangeMask)); - getDataMap().put("timeRange", _timeRangeMask.getDataMap()); - return this; - } - - /** - * Union of [[AbsoluteTimeRange]] and [[RelativeTimeRange]]. - * It indicates the range of input data which is to be loaded. This field generally refers to how much of the input - * data should be restricted using the time in the timestamp column. - * - * For example, - * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from - * 22nd May 2020 to 25th May, 2020 with both dates included. - * We only support yyyyMMdd format for this. In future, if there is a request, we can - * add support for other date time formats as well. - * - * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 - * till 11/04/2020 willl be joined. - * - */ - public InputDataTimeSettings.ProjectionMask withTimeRange() { - _timeRangeMask = null; - getDataMap().put("timeRange", MaskMap.POSITIVE_MASK); - return this; - } - - } - - @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\InputDataTimeSettings.pdl.") - public static class TimeRange - extends UnionTemplate - { - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[absoluteTimeRange:{namespace com.linkedin.feathr.config.join/**The absolute time range with start and end time being required fields.\r\nIt accepts a start time and an end time which should be specifiied using the [[Date.pdl]] or the [[HourTime.pdl]] class.\r\nThis model can be used to represent time range in daily or hourly interval.\r\nabsoluteTimeRange: {\r\n startTime: TimeHour(day=1, month=1, year=2020, hour=13)\r\n endTime: TimeHour(day=3, month=1, year=2020, hour=2)\r\n }\r\n(or)\r\nabsoluteTimeRange: {\r\n startTime: Date(day=1, month=1, year=2020)\r\n endTime: Date(day=3, month=1, year=2020)\r\n }\r\nendTime and startTime should always have the same granularity, ie - Daily or Hourly.\r\nendTme > startTime*/record AbsoluteTimeRange{/**start time of the date range, in daily or hourly format with the start date included in the range.*/startTime:union[date:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}hourTime:/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}]/**end date of the date range, in daily or hourly format with the end date included in the range.*/endTime:union[date:Date,hourTime:HourTime]}}relativeTimeRange:{namespace com.linkedin.feathr.config.join/**The time range represented relative to the current timestamp. It uses the current system time as the reference and can be used to\r\nexpress a range of times with respect to the current time.\r\nExample, - If current time is 01/01/2020, window is 3 days, and offset is 1 day (unit can be day or hour).\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"DAY\"), offset=TimeOffset(length=1, unit=\"Day\"))\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"HOUR\"))*/record RelativeTimeRange{/**Window is the number of time units from the reference time units to look back to obtain the timeRange.\r\nFor example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020\r\ntill 11/05/2020 (both days included).\r\nwindow >= 1 TimeUnit*/window:/**Represents a length of time along with the corresponding time unit (DAY, HOUR).*/record TimeWindow{/**Amount of the duration in TimeUnits. Can be greater or equal to 1.*/@validate.positive,length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}/**Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time.\r\nFor example, if dateOffset is 4, and window is 2 days, then reference time\r\nwill be 4 days ago from today.\r\nExample - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020.\r\nThis will always take the window's timeUnits.*/@validate.integerRange.min=0,offset:long=0}}]", SchemaFormatType.PDL)); - private com.linkedin.feathr.config.join.AbsoluteTimeRange _absoluteTimeRangeMember = null; - private com.linkedin.feathr.config.join.RelativeTimeRange _relativeTimeRangeMember = null; - private InputDataTimeSettings.TimeRange.ChangeListener __changeListener = new InputDataTimeSettings.TimeRange.ChangeListener(this); - private final static DataSchema MEMBER_AbsoluteTimeRange = SCHEMA.getTypeByMemberKey("absoluteTimeRange"); - private final static DataSchema MEMBER_RelativeTimeRange = SCHEMA.getTypeByMemberKey("relativeTimeRange"); - - public TimeRange() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public TimeRange(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static InputDataTimeSettings.TimeRange createWithAbsoluteTimeRange(com.linkedin.feathr.config.join.AbsoluteTimeRange value) { - InputDataTimeSettings.TimeRange newUnion = new InputDataTimeSettings.TimeRange(); - newUnion.setAbsoluteTimeRange(value); - return newUnion; - } - - public boolean isAbsoluteTimeRange() { - return memberIs("absoluteTimeRange"); - } - - public com.linkedin.feathr.config.join.AbsoluteTimeRange getAbsoluteTimeRange() { - checkNotNull(); - if (_absoluteTimeRangeMember!= null) { - return _absoluteTimeRangeMember; - } - Object __rawValue = super._map.get("absoluteTimeRange"); - _absoluteTimeRangeMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.AbsoluteTimeRange(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _absoluteTimeRangeMember; - } - - public void setAbsoluteTimeRange(com.linkedin.feathr.config.join.AbsoluteTimeRange value) { - checkNotNull(); - super._map.clear(); - _absoluteTimeRangeMember = value; - CheckedUtil.putWithoutChecking(super._map, "absoluteTimeRange", value.data()); - } - - public static InputDataTimeSettings.TimeRange createWithRelativeTimeRange(com.linkedin.feathr.config.join.RelativeTimeRange value) { - InputDataTimeSettings.TimeRange newUnion = new InputDataTimeSettings.TimeRange(); - newUnion.setRelativeTimeRange(value); - return newUnion; - } - - public boolean isRelativeTimeRange() { - return memberIs("relativeTimeRange"); - } - - public com.linkedin.feathr.config.join.RelativeTimeRange getRelativeTimeRange() { - checkNotNull(); - if (_relativeTimeRangeMember!= null) { - return _relativeTimeRangeMember; - } - Object __rawValue = super._map.get("relativeTimeRange"); - _relativeTimeRangeMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.RelativeTimeRange(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _relativeTimeRangeMember; - } - - public void setRelativeTimeRange(com.linkedin.feathr.config.join.RelativeTimeRange value) { - checkNotNull(); - super._map.clear(); - _relativeTimeRangeMember = value; - CheckedUtil.putWithoutChecking(super._map, "relativeTimeRange", value.data()); - } - - public static InputDataTimeSettings.TimeRange.ProjectionMask createMask() { - return new InputDataTimeSettings.TimeRange.ProjectionMask(); - } - - @Override - public InputDataTimeSettings.TimeRange clone() - throws CloneNotSupportedException - { - InputDataTimeSettings.TimeRange __clone = ((InputDataTimeSettings.TimeRange) super.clone()); - __clone.__changeListener = new InputDataTimeSettings.TimeRange.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public InputDataTimeSettings.TimeRange copy() - throws CloneNotSupportedException - { - InputDataTimeSettings.TimeRange __copy = ((InputDataTimeSettings.TimeRange) super.copy()); - __copy._relativeTimeRangeMember = null; - __copy._absoluteTimeRangeMember = null; - __copy.__changeListener = new InputDataTimeSettings.TimeRange.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final InputDataTimeSettings.TimeRange __objectRef; - - private ChangeListener(InputDataTimeSettings.TimeRange reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "relativeTimeRange": - __objectRef._relativeTimeRangeMember = null; - break; - case "absoluteTimeRange": - __objectRef._absoluteTimeRangeMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.config.join.AbsoluteTimeRange.Fields AbsoluteTimeRange() { - return new com.linkedin.feathr.config.join.AbsoluteTimeRange.Fields(getPathComponents(), "absoluteTimeRange"); - } - - public com.linkedin.feathr.config.join.RelativeTimeRange.Fields RelativeTimeRange() { - return new com.linkedin.feathr.config.join.RelativeTimeRange.Fields(getPathComponents(), "relativeTimeRange"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.AbsoluteTimeRange.ProjectionMask _AbsoluteTimeRangeMask; - private com.linkedin.feathr.config.join.RelativeTimeRange.ProjectionMask _RelativeTimeRangeMask; - - ProjectionMask() { - super(3); - } - - public InputDataTimeSettings.TimeRange.ProjectionMask withAbsoluteTimeRange(Function nestedMask) { - _AbsoluteTimeRangeMask = nestedMask.apply(((_AbsoluteTimeRangeMask == null)?com.linkedin.feathr.config.join.AbsoluteTimeRange.createMask():_AbsoluteTimeRangeMask)); - getDataMap().put("absoluteTimeRange", _AbsoluteTimeRangeMask.getDataMap()); - return this; - } - - public InputDataTimeSettings.TimeRange.ProjectionMask withRelativeTimeRange(Function nestedMask) { - _RelativeTimeRangeMask = nestedMask.apply(((_RelativeTimeRangeMask == null)?com.linkedin.feathr.config.join.RelativeTimeRange.createMask():_RelativeTimeRangeMask)); - getDataMap().put("relativeTimeRange", _RelativeTimeRangeMask.getDataMap()); - return this; - } - - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoinTimeSettings.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoinTimeSettings.java deleted file mode 100644 index 6c37ba441..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoinTimeSettings.java +++ /dev/null @@ -1,231 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.DataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.TyperefDataSchema; -import com.linkedin.data.schema.UnionDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.HasTyperefInfo; -import com.linkedin.data.template.TyperefInfo; -import com.linkedin.data.template.UnionTemplate; - -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\JoinTimeSettings.pdl.") -public class JoinTimeSettings - extends UnionTemplate - implements HasTyperefInfo -{ - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[useLatestJoinTimeSettings:{namespace com.linkedin.feathr.config.join/**Settings needed when the input data is to be joined with the latest available feature data.\r\njoinTimeSettings: {\r\n useLatestFeatureData: true\r\n}*/record UseLatestJoinTimeSettings{/**Boolean value, if set to true, indicates that the latest available feature data is to be used for joining.\r\nWhen useLatestFeatureData is set, there should be no other time-based parameters.*/useLatestFeatureData:boolean=true}}timestampColJoinTimeSettings:{namespace com.linkedin.feathr.config.join/**Settings needed when the input data has a timestamp which should be used for the join.\r\njoinTimeSettings: {\r\n timestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }\r\n simulateTimeDelay: 1d\r\n}*/record TimestampColJoinTimeSettings{/**The timestamp column name and timeformat which should be used for joining with the feature data.\r\nRefer to [[TimestampColumn]].\r\nExample, TimestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }*/timestampColumn:/**Timestamp column of the input featureiized dataset, which is to be used for the join.\r\ntimestampColumn: {\r\n def: timestamp\r\n format: yyyyMMdd\r\n }*/record TimestampColumn{/**The definiton of the timestamp column, which can be a sql expression involving the timestamp column\r\nor just the column name\r\nExample:- definition: timestamp, timestamp + 10000000.*/definition:union[columnName:string,sparkSqlExpression:/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}]/**Format of the timestamp column. Must confer to java's timestampFormatter or can be\r\nepoch or epoch_millis.\r\nExample:- epoch, epoch_millis, yyyy/MM/dd*/format:/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string}/**An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted\r\nfrom the input data timestamp while joining with the feature data.\r\nWe do support negative time delays.*/simulateTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}}}]", SchemaFormatType.PDL)); - private com.linkedin.feathr.config.join.UseLatestJoinTimeSettings _useLatestJoinTimeSettingsMember = null; - private com.linkedin.feathr.config.join.TimestampColJoinTimeSettings _timestampColJoinTimeSettingsMember = null; - private JoinTimeSettings.ChangeListener __changeListener = new JoinTimeSettings.ChangeListener(this); - private final static DataSchema MEMBER_UseLatestJoinTimeSettings = SCHEMA.getTypeByMemberKey("useLatestJoinTimeSettings"); - private final static DataSchema MEMBER_TimestampColJoinTimeSettings = SCHEMA.getTypeByMemberKey("timestampColJoinTimeSettings"); - private final static TyperefInfo TYPEREFINFO = new JoinTimeSettings.UnionTyperefInfo(); - - public JoinTimeSettings() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public JoinTimeSettings(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static JoinTimeSettings createWithUseLatestJoinTimeSettings(com.linkedin.feathr.config.join.UseLatestJoinTimeSettings value) { - JoinTimeSettings newUnion = new JoinTimeSettings(); - newUnion.setUseLatestJoinTimeSettings(value); - return newUnion; - } - - public boolean isUseLatestJoinTimeSettings() { - return memberIs("useLatestJoinTimeSettings"); - } - - public com.linkedin.feathr.config.join.UseLatestJoinTimeSettings getUseLatestJoinTimeSettings() { - checkNotNull(); - if (_useLatestJoinTimeSettingsMember!= null) { - return _useLatestJoinTimeSettingsMember; - } - Object __rawValue = super._map.get("useLatestJoinTimeSettings"); - _useLatestJoinTimeSettingsMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.UseLatestJoinTimeSettings(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _useLatestJoinTimeSettingsMember; - } - - public void setUseLatestJoinTimeSettings(com.linkedin.feathr.config.join.UseLatestJoinTimeSettings value) { - checkNotNull(); - super._map.clear(); - _useLatestJoinTimeSettingsMember = value; - CheckedUtil.putWithoutChecking(super._map, "useLatestJoinTimeSettings", value.data()); - } - - public static JoinTimeSettings createWithTimestampColJoinTimeSettings(com.linkedin.feathr.config.join.TimestampColJoinTimeSettings value) { - JoinTimeSettings newUnion = new JoinTimeSettings(); - newUnion.setTimestampColJoinTimeSettings(value); - return newUnion; - } - - public boolean isTimestampColJoinTimeSettings() { - return memberIs("timestampColJoinTimeSettings"); - } - - public com.linkedin.feathr.config.join.TimestampColJoinTimeSettings getTimestampColJoinTimeSettings() { - checkNotNull(); - if (_timestampColJoinTimeSettingsMember!= null) { - return _timestampColJoinTimeSettingsMember; - } - Object __rawValue = super._map.get("timestampColJoinTimeSettings"); - _timestampColJoinTimeSettingsMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.TimestampColJoinTimeSettings(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _timestampColJoinTimeSettingsMember; - } - - public void setTimestampColJoinTimeSettings(com.linkedin.feathr.config.join.TimestampColJoinTimeSettings value) { - checkNotNull(); - super._map.clear(); - _timestampColJoinTimeSettingsMember = value; - CheckedUtil.putWithoutChecking(super._map, "timestampColJoinTimeSettings", value.data()); - } - - public static JoinTimeSettings.ProjectionMask createMask() { - return new JoinTimeSettings.ProjectionMask(); - } - - @Override - public JoinTimeSettings clone() - throws CloneNotSupportedException - { - JoinTimeSettings __clone = ((JoinTimeSettings) super.clone()); - __clone.__changeListener = new JoinTimeSettings.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public JoinTimeSettings copy() - throws CloneNotSupportedException - { - JoinTimeSettings __copy = ((JoinTimeSettings) super.copy()); - __copy._useLatestJoinTimeSettingsMember = null; - __copy._timestampColJoinTimeSettingsMember = null; - __copy.__changeListener = new JoinTimeSettings.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - public TyperefInfo typerefInfo() { - return TYPEREFINFO; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final JoinTimeSettings __objectRef; - - private ChangeListener(JoinTimeSettings reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "useLatestJoinTimeSettings": - __objectRef._useLatestJoinTimeSettingsMember = null; - break; - case "timestampColJoinTimeSettings": - __objectRef._timestampColJoinTimeSettingsMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.config.join.UseLatestJoinTimeSettings.Fields UseLatestJoinTimeSettings() { - return new com.linkedin.feathr.config.join.UseLatestJoinTimeSettings.Fields(getPathComponents(), "useLatestJoinTimeSettings"); - } - - public com.linkedin.feathr.config.join.TimestampColJoinTimeSettings.Fields TimestampColJoinTimeSettings() { - return new com.linkedin.feathr.config.join.TimestampColJoinTimeSettings.Fields(getPathComponents(), "timestampColJoinTimeSettings"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.UseLatestJoinTimeSettings.ProjectionMask _UseLatestJoinTimeSettingsMask; - private com.linkedin.feathr.config.join.TimestampColJoinTimeSettings.ProjectionMask _TimestampColJoinTimeSettingsMask; - - ProjectionMask() { - super(3); - } - - public JoinTimeSettings.ProjectionMask withUseLatestJoinTimeSettings(Function nestedMask) { - _UseLatestJoinTimeSettingsMask = nestedMask.apply(((_UseLatestJoinTimeSettingsMask == null)?com.linkedin.feathr.config.join.UseLatestJoinTimeSettings.createMask():_UseLatestJoinTimeSettingsMask)); - getDataMap().put("useLatestJoinTimeSettings", _UseLatestJoinTimeSettingsMask.getDataMap()); - return this; - } - - public JoinTimeSettings.ProjectionMask withTimestampColJoinTimeSettings(Function nestedMask) { - _TimestampColJoinTimeSettingsMask = nestedMask.apply(((_TimestampColJoinTimeSettingsMask == null)?com.linkedin.feathr.config.join.TimestampColJoinTimeSettings.createMask():_TimestampColJoinTimeSettingsMask)); - getDataMap().put("timestampColJoinTimeSettings", _TimestampColJoinTimeSettingsMask.getDataMap()); - return this; - } - - } - - - /** - * JoinTimeSettings contains all the parameters required to join the time sensitive input data with the feature data. - * The input data can be time sensitive in two ways:- - * a. Have a timestamp column - * b. Always join with the latest available feature data. In this case, we do not require a timestamp column. - * c. The file path is time-partition and the path time is used for the join - * (Todo - Add useTimePartitionPattern field in this section) - * In this section, the user needs to let feathr know which of the above properties is to be used for the join. - * - */ - private final static class UnionTyperefInfo - extends TyperefInfo - { - - private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**JoinTimeSettings contains all the parameters required to join the time sensitive input data with the feature data.\r\nThe input data can be time sensitive in two ways:-\r\na. Have a timestamp column\r\nb. Always join with the latest available feature data. In this case, we do not require a timestamp column.\r\nc. The file path is time-partition and the path time is used for the join\r\n(Todo - Add useTimePartitionPattern field in this section)\r\nIn this section, the user needs to let feathr know which of the above properties is to be used for the join.*/typeref JoinTimeSettings=union[useLatestJoinTimeSettings:/**Settings needed when the input data is to be joined with the latest available feature data.\r\njoinTimeSettings: {\r\n useLatestFeatureData: true\r\n}*/record UseLatestJoinTimeSettings{/**Boolean value, if set to true, indicates that the latest available feature data is to be used for joining.\r\nWhen useLatestFeatureData is set, there should be no other time-based parameters.*/useLatestFeatureData:boolean=true}timestampColJoinTimeSettings:/**Settings needed when the input data has a timestamp which should be used for the join.\r\njoinTimeSettings: {\r\n timestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }\r\n simulateTimeDelay: 1d\r\n}*/record TimestampColJoinTimeSettings{/**The timestamp column name and timeformat which should be used for joining with the feature data.\r\nRefer to [[TimestampColumn]].\r\nExample, TimestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }*/timestampColumn:/**Timestamp column of the input featureiized dataset, which is to be used for the join.\r\ntimestampColumn: {\r\n def: timestamp\r\n format: yyyyMMdd\r\n }*/record TimestampColumn{/**The definiton of the timestamp column, which can be a sql expression involving the timestamp column\r\nor just the column name\r\nExample:- definition: timestamp, timestamp + 10000000.*/definition:union[columnName:string,sparkSqlExpression:/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}]/**Format of the timestamp column. Must confer to java's timestampFormatter or can be\r\nepoch or epoch_millis.\r\nExample:- epoch, epoch_millis, yyyy/MM/dd*/format:/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string}/**An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted\r\nfrom the input data timestamp while joining with the feature data.\r\nWe do support negative time delays.*/simulateTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}}]", SchemaFormatType.PDL)); - - public UnionTyperefInfo() { - super(SCHEMA); - } - - public static TyperefDataSchema dataSchema() { - return SCHEMA; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeature.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeature.java deleted file mode 100644 index 6bf8b7a61..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeature.java +++ /dev/null @@ -1,1136 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.DataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.UnionDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; -import com.linkedin.data.template.StringArray; -import com.linkedin.data.template.UnionTemplate; - - -/** - * JoiningFeature is the feature section of the join config. This section consists of information pertaining to a feature - * which is to be joined:- - * a. The join keys of the input data, with which this feature is to be joined. - * b. name of the feature - * c. optional timeRange of the input data which is to be joined with this feature. - * d. optional overrideTimeDelay if this feature needs a different simulate time delay other than the one mentioned. - * - * This is a required section of the the join config. - * Example, - * a. JoiningFeature{ - * keys: ["key1"] - * frameFeatureName: "feature1" - * AbsoluteDateRange(startDate: Date(year=2020, month=5, day=5), - * endDate: Date(year=2020, month=5, day=7)) - * } - * b. JoiningFeature{ - * keys: ["key1"] - * frameFeatureName: "feature2" - * overrideTimeDelay: TimeDelay(length=1, unit="DAY") - * } - * c. JoiningFeature{ - * keys: ["key1"] - * frameFeatureName: "feature3" - * RelativeDateRange(numDays: 5, - * offset: 3) - * } - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\JoiningFeature.pdl.") -public class JoiningFeature - extends RecordTemplate -{ - - private final static JoiningFeature.Fields _fields = new JoiningFeature.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**JoiningFeature is the feature section of the join config. This section consists of information pertaining to a feature\r\nwhich is to be joined:-\r\na. The join keys of the input data, with which this feature is to be joined.\r\nb. name of the feature\r\nc. optional timeRange of the input data which is to be joined with this feature.\r\nd. optional overrideTimeDelay if this feature needs a different simulate time delay other than the one mentioned.\r\n\r\nThis is a required section of the the join config.\r\nExample,\r\n a. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature1\"\r\n AbsoluteDateRange(startDate: Date(year=2020, month=5, day=5),\r\n endDate: Date(year=2020, month=5, day=7))\r\n }\r\n b. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature2\"\r\n overrideTimeDelay: TimeDelay(length=1, unit=\"DAY\")\r\n }\r\n c. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature3\"\r\n RelativeDateRange(numDays: 5,\r\n offset: 3)\r\n }*/record JoiningFeature{/**Keys to join input with feature source, the field name of the key in the input featuized dataset.*/keys:array[string]/**Feature name as defined in feathr's feature definition configuration.\r\n\r\nCurrently the column in the output FDS that holds this feature will have the same name as feature name.\r\nIf multiple joined features have the same name and no alias is defined for them, feathr will prepend the keys to the feature name.\r\n\r\nIn the future, if \"featureAlias\" is not set, the column in the output FDS that holds this feature will have the same name as feature name.\r\nIf multiple joined features have the same name and no alias is defined for them, the join operation will fail\r\n(to avoid produciing two columns in the output FDS with the same name).*/frameFeatureName:string/**The development of this is in progress. This is not in use for now.\r\n\r\nThe name to be used for the column in the output FDS that contains the values from this joined feature.\r\nIf not set, the name of the feature (frameFeatureName) will be used for the output column.\r\nFor example, if the user request joining a feature named \"careers_job_listTime\" and provides no alias,\r\nthe output FDS will contain a column called \"careers_job_listTime\". However, if the user sets \"featureAlias\" to \"list_time\",\r\nthe column will be named \"list_time\".\r\n\r\nfeature alias can be useful for in a few cases:\r\n - If the user prefers to use a name different than the feathr name in their model,\r\nthey can use an alias to control the name of the column in the output FDS.\r\n - Sometimes, the training datas needs to have two features that are from the same feathr feature.\r\nFor example, if we are modeing the problem of the probability of a member A (viewer) seeing the profile of member B\r\n(viewee) and we want to use the skills of both viewer and viewee as features, we need to join feathr feature\r\n\"member_skills\" of member A with feathr feature \"member_skills\" of member B. That is, the two features are the same\r\nfeature but for different entiity ids). The default behavior of join is to name the output column name using the feathr\r\nfeature name, but in a case like the above case, that would result in two columns with the same name,\r\nwhich is not valid for FDS. In these cases, the user has to provide an alias for at least one of these joined features.\r\nFor example, the user can use featureAliases such as \"viewer_skills\" and \"viewee_skills\".\r\nIn these cases, featureAliases becomes mandatory.*/featureAlias:optional string/**dateRange is used in Time-based joins, which refers to the situation when one or multiple days of input data needs\r\nto be used for training.\r\nOne of the common use cases where this is used, is in training with some time-insensitive features, or\r\ntraining pipeline that always use the full day data, one day before running (since there is only partial data for today).\r\nThe time for the input featurized dataset can be set using this field.\r\nHourly data is not allowed in this case.\r\n\r\nFor example,\r\na. startDate: \"20200522\", endDate: \"20200525\" implies this feature should be joined with the input data starting from\r\n22nd May 2020 to 25th May, 2020 with both dates included.\r\nWe only support yyyyMMdd format for this. In future, if there is a request, we can\r\nadd support for other date time formats as well.\r\n\r\nb. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020\r\ntill 11/04/2020 willl be joined.\r\n\r\nP.S - This is different from the timeRange used in settings as the settings startTime is applicable for the entire input data,\r\nwhile this a feature level setting. Also, we do not support hourly time here.*/dateRange:optional union[absoluteDateRange:/**The absolute date range with start and end date being required fields.\r\nIt accepts a start date and an end date which should be specifiied using the [[Date.pdl]] class.\r\nabsoluteDateRange: {\r\n startDate: Date(day=1, month=1, year=2020)\r\n endDate: Date(day=3, month=1, year=2020)\r\n }\r\n In this case, the endDate > startDate.*/record AbsoluteDateRange{/**start date of the date range, with the start date included in the range.*/startDate:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}/**end date of the date range, with the end date included in the range.*/endDate:Date}relativeDateRange:/**The date range represented relative to the current date. It uses the current system date as the reference and can be used to\r\nexpress a range of dates with respect to the current date.\r\nExample, - If current date is 01/01/2020, window is 3, and offset 1 (unit is number of days)\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nIf dateOffset is not specified, it defaults to 0.\r\nrelativeDateRange: RelativeDateRange(numDays=2, dateOffset=1)\r\nrelativeDateRange: RelativeDateRange(numDays=5)*/record RelativeDateRange{/**Represents a length of time.\r\nnumDays is the window from the reference date to look back to obtain a dateRange.\r\nFor example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020\r\ntill 11/05/2020.*/@validate.positive={}numDays:long/**Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date\r\nwill be 4 days ago from today.*/dateOffset:long=0}]/**The override time delay parameter which will override the global simulate time delay specified in the settings section for\r\nthe particular feature.\r\nThis parameter is only applicable when the simulate time delay is set in the settings section\r\nFor example, let us say the global simulate delay was 5d, and the overrideTimeDelay is set to 3d.\r\nThen, for this specificc feature, a simulate delay of 3d will be applied.*/overrideTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}}", SchemaFormatType.PDL)); - private StringArray _keysField = null; - private String _frameFeatureNameField = null; - private String _featureAliasField = null; - private JoiningFeature.DateRange _dateRangeField = null; - private TimeOffset _overrideTimeDelayField = null; - private JoiningFeature.ChangeListener __changeListener = new JoiningFeature.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Keys = SCHEMA.getField("keys"); - private final static RecordDataSchema.Field FIELD_FrameFeatureName = SCHEMA.getField("frameFeatureName"); - private final static RecordDataSchema.Field FIELD_FeatureAlias = SCHEMA.getField("featureAlias"); - private final static RecordDataSchema.Field FIELD_DateRange = SCHEMA.getField("dateRange"); - private final static RecordDataSchema.Field FIELD_OverrideTimeDelay = SCHEMA.getField("overrideTimeDelay"); - - public JoiningFeature() { - super(new DataMap(7, 0.75F), SCHEMA, 4); - addChangeListener(__changeListener); - } - - public JoiningFeature(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static JoiningFeature.Fields fields() { - return _fields; - } - - public static JoiningFeature.ProjectionMask createMask() { - return new JoiningFeature.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for keys - * - * @see JoiningFeature.Fields#keys - */ - public boolean hasKeys() { - if (_keysField!= null) { - return true; - } - return super._map.containsKey("keys"); - } - - /** - * Remover for keys - * - * @see JoiningFeature.Fields#keys - */ - public void removeKeys() { - super._map.remove("keys"); - } - - /** - * Getter for keys - * - * @see JoiningFeature.Fields#keys - */ - public StringArray getKeys(GetMode mode) { - switch (mode) { - case STRICT: - return getKeys(); - case DEFAULT: - case NULL: - if (_keysField!= null) { - return _keysField; - } else { - Object __rawValue = super._map.get("keys"); - _keysField = ((__rawValue == null)?null:new StringArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _keysField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for keys - * - * @return - * Required field. Could be null for partial record. - * @see JoiningFeature.Fields#keys - */ - @Nonnull - public StringArray getKeys() { - if (_keysField!= null) { - return _keysField; - } else { - Object __rawValue = super._map.get("keys"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("keys"); - } - _keysField = ((__rawValue == null)?null:new StringArray(DataTemplateUtil.castOrThrow(__rawValue, DataList.class))); - return _keysField; - } - } - - /** - * Setter for keys - * - * @see JoiningFeature.Fields#keys - */ - public JoiningFeature setKeys(StringArray value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setKeys(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field keys of com.linkedin.feathr.config.join.JoiningFeature"); - } else { - CheckedUtil.putWithoutChecking(super._map, "keys", value.data()); - _keysField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeKeys(); - } else { - CheckedUtil.putWithoutChecking(super._map, "keys", value.data()); - _keysField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "keys", value.data()); - _keysField = value; - } - break; - } - return this; - } - - /** - * Setter for keys - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see JoiningFeature.Fields#keys - */ - public JoiningFeature setKeys( - @Nonnull - StringArray value) { - if (value == null) { - throw new NullPointerException("Cannot set field keys of com.linkedin.feathr.config.join.JoiningFeature to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "keys", value.data()); - _keysField = value; - } - return this; - } - - /** - * Existence checker for frameFeatureName - * - * @see JoiningFeature.Fields#frameFeatureName - */ - public boolean hasFrameFeatureName() { - if (_frameFeatureNameField!= null) { - return true; - } - return super._map.containsKey("frameFeatureName"); - } - - /** - * Remover for frameFeatureName - * - * @see JoiningFeature.Fields#frameFeatureName - */ - public void removeFrameFeatureName() { - super._map.remove("frameFeatureName"); - } - - /** - * Getter for frameFeatureName - * - * @see JoiningFeature.Fields#frameFeatureName - */ - public String getFrameFeatureName(GetMode mode) { - switch (mode) { - case STRICT: - return getFrameFeatureName(); - case DEFAULT: - case NULL: - if (_frameFeatureNameField!= null) { - return _frameFeatureNameField; - } else { - Object __rawValue = super._map.get("frameFeatureName"); - _frameFeatureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _frameFeatureNameField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for frameFeatureName - * - * @return - * Required field. Could be null for partial record. - * @see JoiningFeature.Fields#frameFeatureName - */ - @Nonnull - public String getFrameFeatureName() { - if (_frameFeatureNameField!= null) { - return _frameFeatureNameField; - } else { - Object __rawValue = super._map.get("frameFeatureName"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("frameFeatureName"); - } - _frameFeatureNameField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _frameFeatureNameField; - } - } - - /** - * Setter for frameFeatureName - * - * @see JoiningFeature.Fields#frameFeatureName - */ - public JoiningFeature setFrameFeatureName(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFrameFeatureName(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field frameFeatureName of com.linkedin.feathr.config.join.JoiningFeature"); - } else { - CheckedUtil.putWithoutChecking(super._map, "frameFeatureName", value); - _frameFeatureNameField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeFrameFeatureName(); - } else { - CheckedUtil.putWithoutChecking(super._map, "frameFeatureName", value); - _frameFeatureNameField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "frameFeatureName", value); - _frameFeatureNameField = value; - } - break; - } - return this; - } - - /** - * Setter for frameFeatureName - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see JoiningFeature.Fields#frameFeatureName - */ - public JoiningFeature setFrameFeatureName( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field frameFeatureName of com.linkedin.feathr.config.join.JoiningFeature to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "frameFeatureName", value); - _frameFeatureNameField = value; - } - return this; - } - - /** - * Existence checker for featureAlias - * - * @see JoiningFeature.Fields#featureAlias - */ - public boolean hasFeatureAlias() { - if (_featureAliasField!= null) { - return true; - } - return super._map.containsKey("featureAlias"); - } - - /** - * Remover for featureAlias - * - * @see JoiningFeature.Fields#featureAlias - */ - public void removeFeatureAlias() { - super._map.remove("featureAlias"); - } - - /** - * Getter for featureAlias - * - * @see JoiningFeature.Fields#featureAlias - */ - public String getFeatureAlias(GetMode mode) { - return getFeatureAlias(); - } - - /** - * Getter for featureAlias - * - * @return - * Optional field. Always check for null. - * @see JoiningFeature.Fields#featureAlias - */ - @Nullable - public String getFeatureAlias() { - if (_featureAliasField!= null) { - return _featureAliasField; - } else { - Object __rawValue = super._map.get("featureAlias"); - _featureAliasField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _featureAliasField; - } - } - - /** - * Setter for featureAlias - * - * @see JoiningFeature.Fields#featureAlias - */ - public JoiningFeature setFeatureAlias(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFeatureAlias(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeFeatureAlias(); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureAlias", value); - _featureAliasField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "featureAlias", value); - _featureAliasField = value; - } - break; - } - return this; - } - - /** - * Setter for featureAlias - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see JoiningFeature.Fields#featureAlias - */ - public JoiningFeature setFeatureAlias( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field featureAlias of com.linkedin.feathr.config.join.JoiningFeature to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "featureAlias", value); - _featureAliasField = value; - } - return this; - } - - /** - * Existence checker for dateRange - * - * @see JoiningFeature.Fields#dateRange - */ - public boolean hasDateRange() { - if (_dateRangeField!= null) { - return true; - } - return super._map.containsKey("dateRange"); - } - - /** - * Remover for dateRange - * - * @see JoiningFeature.Fields#dateRange - */ - public void removeDateRange() { - super._map.remove("dateRange"); - } - - /** - * Getter for dateRange - * - * @see JoiningFeature.Fields#dateRange - */ - public JoiningFeature.DateRange getDateRange(GetMode mode) { - return getDateRange(); - } - - /** - * Getter for dateRange - * - * @return - * Optional field. Always check for null. - * @see JoiningFeature.Fields#dateRange - */ - @Nullable - public JoiningFeature.DateRange getDateRange() { - if (_dateRangeField!= null) { - return _dateRangeField; - } else { - Object __rawValue = super._map.get("dateRange"); - _dateRangeField = ((__rawValue == null)?null:new JoiningFeature.DateRange(__rawValue)); - return _dateRangeField; - } - } - - /** - * Setter for dateRange - * - * @see JoiningFeature.Fields#dateRange - */ - public JoiningFeature setDateRange(JoiningFeature.DateRange value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setDateRange(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeDateRange(); - } else { - CheckedUtil.putWithoutChecking(super._map, "dateRange", value.data()); - _dateRangeField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "dateRange", value.data()); - _dateRangeField = value; - } - break; - } - return this; - } - - /** - * Setter for dateRange - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see JoiningFeature.Fields#dateRange - */ - public JoiningFeature setDateRange( - @Nonnull - JoiningFeature.DateRange value) { - if (value == null) { - throw new NullPointerException("Cannot set field dateRange of com.linkedin.feathr.config.join.JoiningFeature to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "dateRange", value.data()); - _dateRangeField = value; - } - return this; - } - - /** - * Existence checker for overrideTimeDelay - * - * @see JoiningFeature.Fields#overrideTimeDelay - */ - public boolean hasOverrideTimeDelay() { - if (_overrideTimeDelayField!= null) { - return true; - } - return super._map.containsKey("overrideTimeDelay"); - } - - /** - * Remover for overrideTimeDelay - * - * @see JoiningFeature.Fields#overrideTimeDelay - */ - public void removeOverrideTimeDelay() { - super._map.remove("overrideTimeDelay"); - } - - /** - * Getter for overrideTimeDelay - * - * @see JoiningFeature.Fields#overrideTimeDelay - */ - public TimeOffset getOverrideTimeDelay(GetMode mode) { - return getOverrideTimeDelay(); - } - - /** - * Getter for overrideTimeDelay - * - * @return - * Optional field. Always check for null. - * @see JoiningFeature.Fields#overrideTimeDelay - */ - @Nullable - public TimeOffset getOverrideTimeDelay() { - if (_overrideTimeDelayField!= null) { - return _overrideTimeDelayField; - } else { - Object __rawValue = super._map.get("overrideTimeDelay"); - _overrideTimeDelayField = ((__rawValue == null)?null:new TimeOffset(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _overrideTimeDelayField; - } - } - - /** - * Setter for overrideTimeDelay - * - * @see JoiningFeature.Fields#overrideTimeDelay - */ - public JoiningFeature setOverrideTimeDelay(TimeOffset value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setOverrideTimeDelay(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeOverrideTimeDelay(); - } else { - CheckedUtil.putWithoutChecking(super._map, "overrideTimeDelay", value.data()); - _overrideTimeDelayField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "overrideTimeDelay", value.data()); - _overrideTimeDelayField = value; - } - break; - } - return this; - } - - /** - * Setter for overrideTimeDelay - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see JoiningFeature.Fields#overrideTimeDelay - */ - public JoiningFeature setOverrideTimeDelay( - @Nonnull - TimeOffset value) { - if (value == null) { - throw new NullPointerException("Cannot set field overrideTimeDelay of com.linkedin.feathr.config.join.JoiningFeature to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "overrideTimeDelay", value.data()); - _overrideTimeDelayField = value; - } - return this; - } - - @Override - public JoiningFeature clone() - throws CloneNotSupportedException - { - JoiningFeature __clone = ((JoiningFeature) super.clone()); - __clone.__changeListener = new JoiningFeature.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public JoiningFeature copy() - throws CloneNotSupportedException - { - JoiningFeature __copy = ((JoiningFeature) super.copy()); - __copy._overrideTimeDelayField = null; - __copy._featureAliasField = null; - __copy._dateRangeField = null; - __copy._keysField = null; - __copy._frameFeatureNameField = null; - __copy.__changeListener = new JoiningFeature.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final JoiningFeature __objectRef; - - private ChangeListener(JoiningFeature reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "overrideTimeDelay": - __objectRef._overrideTimeDelayField = null; - break; - case "featureAlias": - __objectRef._featureAliasField = null; - break; - case "dateRange": - __objectRef._dateRangeField = null; - break; - case "keys": - __objectRef._keysField = null; - break; - case "frameFeatureName": - __objectRef._frameFeatureNameField = null; - break; - } - } - - } - - @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\JoiningFeature.pdl.") - public static class DateRange - extends UnionTemplate - { - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[absoluteDateRange:{namespace com.linkedin.feathr.config.join/**The absolute date range with start and end date being required fields.\r\nIt accepts a start date and an end date which should be specifiied using the [[Date.pdl]] class.\r\nabsoluteDateRange: {\r\n startDate: Date(day=1, month=1, year=2020)\r\n endDate: Date(day=3, month=1, year=2020)\r\n }\r\n In this case, the endDate > startDate.*/record AbsoluteDateRange{/**start date of the date range, with the start date included in the range.*/startDate:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}/**end date of the date range, with the end date included in the range.*/endDate:Date}}relativeDateRange:{namespace com.linkedin.feathr.config.join/**The date range represented relative to the current date. It uses the current system date as the reference and can be used to\r\nexpress a range of dates with respect to the current date.\r\nExample, - If current date is 01/01/2020, window is 3, and offset 1 (unit is number of days)\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nIf dateOffset is not specified, it defaults to 0.\r\nrelativeDateRange: RelativeDateRange(numDays=2, dateOffset=1)\r\nrelativeDateRange: RelativeDateRange(numDays=5)*/record RelativeDateRange{/**Represents a length of time.\r\nnumDays is the window from the reference date to look back to obtain a dateRange.\r\nFor example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020\r\ntill 11/05/2020.*/@validate.positive={}numDays:long/**Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date\r\nwill be 4 days ago from today.*/dateOffset:long=0}}]", SchemaFormatType.PDL)); - private com.linkedin.feathr.config.join.AbsoluteDateRange _absoluteDateRangeMember = null; - private com.linkedin.feathr.config.join.RelativeDateRange _relativeDateRangeMember = null; - private JoiningFeature.DateRange.ChangeListener __changeListener = new JoiningFeature.DateRange.ChangeListener(this); - private final static DataSchema MEMBER_AbsoluteDateRange = SCHEMA.getTypeByMemberKey("absoluteDateRange"); - private final static DataSchema MEMBER_RelativeDateRange = SCHEMA.getTypeByMemberKey("relativeDateRange"); - - public DateRange() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public DateRange(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static JoiningFeature.DateRange createWithAbsoluteDateRange(com.linkedin.feathr.config.join.AbsoluteDateRange value) { - JoiningFeature.DateRange newUnion = new JoiningFeature.DateRange(); - newUnion.setAbsoluteDateRange(value); - return newUnion; - } - - public boolean isAbsoluteDateRange() { - return memberIs("absoluteDateRange"); - } - - public com.linkedin.feathr.config.join.AbsoluteDateRange getAbsoluteDateRange() { - checkNotNull(); - if (_absoluteDateRangeMember!= null) { - return _absoluteDateRangeMember; - } - Object __rawValue = super._map.get("absoluteDateRange"); - _absoluteDateRangeMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.AbsoluteDateRange(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _absoluteDateRangeMember; - } - - public void setAbsoluteDateRange(com.linkedin.feathr.config.join.AbsoluteDateRange value) { - checkNotNull(); - super._map.clear(); - _absoluteDateRangeMember = value; - CheckedUtil.putWithoutChecking(super._map, "absoluteDateRange", value.data()); - } - - public static JoiningFeature.DateRange createWithRelativeDateRange(com.linkedin.feathr.config.join.RelativeDateRange value) { - JoiningFeature.DateRange newUnion = new JoiningFeature.DateRange(); - newUnion.setRelativeDateRange(value); - return newUnion; - } - - public boolean isRelativeDateRange() { - return memberIs("relativeDateRange"); - } - - public com.linkedin.feathr.config.join.RelativeDateRange getRelativeDateRange() { - checkNotNull(); - if (_relativeDateRangeMember!= null) { - return _relativeDateRangeMember; - } - Object __rawValue = super._map.get("relativeDateRange"); - _relativeDateRangeMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.RelativeDateRange(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _relativeDateRangeMember; - } - - public void setRelativeDateRange(com.linkedin.feathr.config.join.RelativeDateRange value) { - checkNotNull(); - super._map.clear(); - _relativeDateRangeMember = value; - CheckedUtil.putWithoutChecking(super._map, "relativeDateRange", value.data()); - } - - public static JoiningFeature.DateRange.ProjectionMask createMask() { - return new JoiningFeature.DateRange.ProjectionMask(); - } - - @Override - public JoiningFeature.DateRange clone() - throws CloneNotSupportedException - { - JoiningFeature.DateRange __clone = ((JoiningFeature.DateRange) super.clone()); - __clone.__changeListener = new JoiningFeature.DateRange.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public JoiningFeature.DateRange copy() - throws CloneNotSupportedException - { - JoiningFeature.DateRange __copy = ((JoiningFeature.DateRange) super.copy()); - __copy._absoluteDateRangeMember = null; - __copy._relativeDateRangeMember = null; - __copy.__changeListener = new JoiningFeature.DateRange.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final JoiningFeature.DateRange __objectRef; - - private ChangeListener(JoiningFeature.DateRange reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "absoluteDateRange": - __objectRef._absoluteDateRangeMember = null; - break; - case "relativeDateRange": - __objectRef._relativeDateRangeMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.config.join.AbsoluteDateRange.Fields AbsoluteDateRange() { - return new com.linkedin.feathr.config.join.AbsoluteDateRange.Fields(getPathComponents(), "absoluteDateRange"); - } - - public com.linkedin.feathr.config.join.RelativeDateRange.Fields RelativeDateRange() { - return new com.linkedin.feathr.config.join.RelativeDateRange.Fields(getPathComponents(), "relativeDateRange"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.AbsoluteDateRange.ProjectionMask _AbsoluteDateRangeMask; - private com.linkedin.feathr.config.join.RelativeDateRange.ProjectionMask _RelativeDateRangeMask; - - ProjectionMask() { - super(3); - } - - public JoiningFeature.DateRange.ProjectionMask withAbsoluteDateRange(Function nestedMask) { - _AbsoluteDateRangeMask = nestedMask.apply(((_AbsoluteDateRangeMask == null)?com.linkedin.feathr.config.join.AbsoluteDateRange.createMask():_AbsoluteDateRangeMask)); - getDataMap().put("absoluteDateRange", _AbsoluteDateRangeMask.getDataMap()); - return this; - } - - public JoiningFeature.DateRange.ProjectionMask withRelativeDateRange(Function nestedMask) { - _RelativeDateRangeMask = nestedMask.apply(((_RelativeDateRangeMask == null)?com.linkedin.feathr.config.join.RelativeDateRange.createMask():_RelativeDateRangeMask)); - getDataMap().put("relativeDateRange", _RelativeDateRangeMask.getDataMap()); - return this; - } - - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Keys to join input with feature source, the field name of the key in the input featuized dataset. - * - */ - public PathSpec keys() { - return new PathSpec(getPathComponents(), "keys"); - } - - /** - * Keys to join input with feature source, the field name of the key in the input featuized dataset. - * - */ - public PathSpec keys(Integer start, Integer count) { - PathSpec arrayPathSpec = new PathSpec(getPathComponents(), "keys"); - if (start!= null) { - arrayPathSpec.setAttribute("start", start); - } - if (count!= null) { - arrayPathSpec.setAttribute("count", count); - } - return arrayPathSpec; - } - - /** - * Feature name as defined in feathr's feature definition configuration. - * - * Currently the column in the output FDS that holds this feature will have the same name as feature name. - * If multiple joined features have the same name and no alias is defined for them, feathr will prepend the keys to the feature name. - * - * In the future, if "featureAlias" is not set, the column in the output FDS that holds this feature will have the same name as feature name. - * If multiple joined features have the same name and no alias is defined for them, the join operation will fail - * (to avoid produciing two columns in the output FDS with the same name). - * - */ - public PathSpec frameFeatureName() { - return new PathSpec(getPathComponents(), "frameFeatureName"); - } - - /** - * The development of this is in progress. This is not in use for now. - * - * The name to be used for the column in the output FDS that contains the values from this joined feature. - * If not set, the name of the feature (frameFeatureName) will be used for the output column. - * For example, if the user request joining a feature named "careers_job_listTime" and provides no alias, - * the output FDS will contain a column called "careers_job_listTime". However, if the user sets "featureAlias" to "list_time", - * the column will be named "list_time". - * - * feature alias can be useful for in a few cases: - * - If the user prefers to use a name different than the feathr name in their model, - * they can use an alias to control the name of the column in the output FDS. - * - Sometimes, the training datas needs to have two features that are from the same feathr feature. - * For example, if we are modeing the problem of the probability of a member A (viewer) seeing the profile of member B - * (viewee) and we want to use the skills of both viewer and viewee as features, we need to join feathr feature - * "member_skills" of member A with feathr feature "member_skills" of member B. That is, the two features are the same - * feature but for different entiity ids). The default behavior of join is to name the output column name using the feathr - * feature name, but in a case like the above case, that would result in two columns with the same name, - * which is not valid for FDS. In these cases, the user has to provide an alias for at least one of these joined features. - * For example, the user can use featureAliases such as "viewer_skills" and "viewee_skills". - * In these cases, featureAliases becomes mandatory. - * - */ - public PathSpec featureAlias() { - return new PathSpec(getPathComponents(), "featureAlias"); - } - - /** - * dateRange is used in Time-based joins, which refers to the situation when one or multiple days of input data needs - * to be used for training. - * One of the common use cases where this is used, is in training with some time-insensitive features, or - * training pipeline that always use the full day data, one day before running (since there is only partial data for today). - * The time for the input featurized dataset can be set using this field. - * Hourly data is not allowed in this case. - * - * For example, - * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from - * 22nd May 2020 to 25th May, 2020 with both dates included. - * We only support yyyyMMdd format for this. In future, if there is a request, we can - * add support for other date time formats as well. - * - * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 - * till 11/04/2020 willl be joined. - * - * P.S - This is different from the timeRange used in settings as the settings startTime is applicable for the entire input data, - * while this a feature level setting. Also, we do not support hourly time here. - * - */ - public com.linkedin.feathr.config.join.JoiningFeature.DateRange.Fields dateRange() { - return new com.linkedin.feathr.config.join.JoiningFeature.DateRange.Fields(getPathComponents(), "dateRange"); - } - - /** - * The override time delay parameter which will override the global simulate time delay specified in the settings section for - * the particular feature. - * This parameter is only applicable when the simulate time delay is set in the settings section - * For example, let us say the global simulate delay was 5d, and the overrideTimeDelay is set to 3d. - * Then, for this specificc feature, a simulate delay of 3d will be applied. - * - */ - public com.linkedin.feathr.config.join.TimeOffset.Fields overrideTimeDelay() { - return new com.linkedin.feathr.config.join.TimeOffset.Fields(getPathComponents(), "overrideTimeDelay"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.JoiningFeature.DateRange.ProjectionMask _dateRangeMask; - private com.linkedin.feathr.config.join.TimeOffset.ProjectionMask _overrideTimeDelayMask; - - ProjectionMask() { - super(7); - } - - /** - * Keys to join input with feature source, the field name of the key in the input featuized dataset. - * - */ - public JoiningFeature.ProjectionMask withKeys() { - getDataMap().put("keys", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Keys to join input with feature source, the field name of the key in the input featuized dataset. - * - */ - public JoiningFeature.ProjectionMask withKeys(Integer start, Integer count) { - getDataMap().put("keys", new DataMap(3)); - if (start!= null) { - getDataMap().getDataMap("keys").put("$start", start); - } - if (count!= null) { - getDataMap().getDataMap("keys").put("$count", count); - } - return this; - } - - /** - * Feature name as defined in feathr's feature definition configuration. - * - * Currently the column in the output FDS that holds this feature will have the same name as feature name. - * If multiple joined features have the same name and no alias is defined for them, feathr will prepend the keys to the feature name. - * - * In the future, if "featureAlias" is not set, the column in the output FDS that holds this feature will have the same name as feature name. - * If multiple joined features have the same name and no alias is defined for them, the join operation will fail - * (to avoid produciing two columns in the output FDS with the same name). - * - */ - public JoiningFeature.ProjectionMask withFrameFeatureName() { - getDataMap().put("frameFeatureName", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * The development of this is in progress. This is not in use for now. - * - * The name to be used for the column in the output FDS that contains the values from this joined feature. - * If not set, the name of the feature (frameFeatureName) will be used for the output column. - * For example, if the user request joining a feature named "careers_job_listTime" and provides no alias, - * the output FDS will contain a column called "careers_job_listTime". However, if the user sets "featureAlias" to "list_time", - * the column will be named "list_time". - * - * feature alias can be useful for in a few cases: - * - If the user prefers to use a name different than the feathr name in their model, - * they can use an alias to control the name of the column in the output FDS. - * - Sometimes, the training datas needs to have two features that are from the same feathr feature. - * For example, if we are modeing the problem of the probability of a member A (viewer) seeing the profile of member B - * (viewee) and we want to use the skills of both viewer and viewee as features, we need to join feathr feature - * "member_skills" of member A with feathr feature "member_skills" of member B. That is, the two features are the same - * feature but for different entiity ids). The default behavior of join is to name the output column name using the feathr - * feature name, but in a case like the above case, that would result in two columns with the same name, - * which is not valid for FDS. In these cases, the user has to provide an alias for at least one of these joined features. - * For example, the user can use featureAliases such as "viewer_skills" and "viewee_skills". - * In these cases, featureAliases becomes mandatory. - * - */ - public JoiningFeature.ProjectionMask withFeatureAlias() { - getDataMap().put("featureAlias", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * dateRange is used in Time-based joins, which refers to the situation when one or multiple days of input data needs - * to be used for training. - * One of the common use cases where this is used, is in training with some time-insensitive features, or - * training pipeline that always use the full day data, one day before running (since there is only partial data for today). - * The time for the input featurized dataset can be set using this field. - * Hourly data is not allowed in this case. - * - * For example, - * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from - * 22nd May 2020 to 25th May, 2020 with both dates included. - * We only support yyyyMMdd format for this. In future, if there is a request, we can - * add support for other date time formats as well. - * - * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 - * till 11/04/2020 willl be joined. - * - * P.S - This is different from the timeRange used in settings as the settings startTime is applicable for the entire input data, - * while this a feature level setting. Also, we do not support hourly time here. - * - */ - public JoiningFeature.ProjectionMask withDateRange(Function nestedMask) { - _dateRangeMask = nestedMask.apply(((_dateRangeMask == null)?JoiningFeature.DateRange.createMask():_dateRangeMask)); - getDataMap().put("dateRange", _dateRangeMask.getDataMap()); - return this; - } - - /** - * dateRange is used in Time-based joins, which refers to the situation when one or multiple days of input data needs - * to be used for training. - * One of the common use cases where this is used, is in training with some time-insensitive features, or - * training pipeline that always use the full day data, one day before running (since there is only partial data for today). - * The time for the input featurized dataset can be set using this field. - * Hourly data is not allowed in this case. - * - * For example, - * a. startDate: "20200522", endDate: "20200525" implies this feature should be joined with the input data starting from - * 22nd May 2020 to 25th May, 2020 with both dates included. - * We only support yyyyMMdd format for this. In future, if there is a request, we can - * add support for other date time formats as well. - * - * b. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020 - * till 11/04/2020 willl be joined. - * - * P.S - This is different from the timeRange used in settings as the settings startTime is applicable for the entire input data, - * while this a feature level setting. Also, we do not support hourly time here. - * - */ - public JoiningFeature.ProjectionMask withDateRange() { - _dateRangeMask = null; - getDataMap().put("dateRange", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * The override time delay parameter which will override the global simulate time delay specified in the settings section for - * the particular feature. - * This parameter is only applicable when the simulate time delay is set in the settings section - * For example, let us say the global simulate delay was 5d, and the overrideTimeDelay is set to 3d. - * Then, for this specificc feature, a simulate delay of 3d will be applied. - * - */ - public JoiningFeature.ProjectionMask withOverrideTimeDelay(Function nestedMask) { - _overrideTimeDelayMask = nestedMask.apply(((_overrideTimeDelayMask == null)?TimeOffset.createMask():_overrideTimeDelayMask)); - getDataMap().put("overrideTimeDelay", _overrideTimeDelayMask.getDataMap()); - return this; - } - - /** - * The override time delay parameter which will override the global simulate time delay specified in the settings section for - * the particular feature. - * This parameter is only applicable when the simulate time delay is set in the settings section - * For example, let us say the global simulate delay was 5d, and the overrideTimeDelay is set to 3d. - * Then, for this specificc feature, a simulate delay of 3d will be applied. - * - */ - public JoiningFeature.ProjectionMask withOverrideTimeDelay() { - _overrideTimeDelayMask = null; - getDataMap().put("overrideTimeDelay", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeatureArray.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeatureArray.java deleted file mode 100644 index 0f8368c02..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/JoiningFeatureArray.java +++ /dev/null @@ -1,118 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import com.linkedin.data.DataList; -import com.linkedin.data.DataMap; -import com.linkedin.data.schema.ArrayDataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.TemplateOutputCastException; -import com.linkedin.data.template.WrappingArrayTemplate; - -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\FrameFeatureJoinConfig.pdl.") -public class JoiningFeatureArray - extends WrappingArrayTemplate -{ - - private final static ArrayDataSchema SCHEMA = ((ArrayDataSchema) DataTemplateUtil.parseSchema("array[{namespace com.linkedin.feathr.config.join/**JoiningFeature is the feature section of the join config. This section consists of information pertaining to a feature\r\nwhich is to be joined:-\r\na. The join keys of the input data, with which this feature is to be joined.\r\nb. name of the feature\r\nc. optional timeRange of the input data which is to be joined with this feature.\r\nd. optional overrideTimeDelay if this feature needs a different simulate time delay other than the one mentioned.\r\n\r\nThis is a required section of the the join config.\r\nExample,\r\n a. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature1\"\r\n AbsoluteDateRange(startDate: Date(year=2020, month=5, day=5),\r\n endDate: Date(year=2020, month=5, day=7))\r\n }\r\n b. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature2\"\r\n overrideTimeDelay: TimeDelay(length=1, unit=\"DAY\")\r\n }\r\n c. JoiningFeature{\r\n keys: [\"key1\"]\r\n frameFeatureName: \"feature3\"\r\n RelativeDateRange(numDays: 5,\r\n offset: 3)\r\n }*/record JoiningFeature{/**Keys to join input with feature source, the field name of the key in the input featuized dataset.*/keys:array[string]/**Feature name as defined in feathr's feature definition configuration.\r\n\r\nCurrently the column in the output FDS that holds this feature will have the same name as feature name.\r\nIf multiple joined features have the same name and no alias is defined for them, feathr will prepend the keys to the feature name.\r\n\r\nIn the future, if \"featureAlias\" is not set, the column in the output FDS that holds this feature will have the same name as feature name.\r\nIf multiple joined features have the same name and no alias is defined for them, the join operation will fail\r\n(to avoid produciing two columns in the output FDS with the same name).*/frameFeatureName:string/**The development of this is in progress. This is not in use for now.\r\n\r\nThe name to be used for the column in the output FDS that contains the values from this joined feature.\r\nIf not set, the name of the feature (frameFeatureName) will be used for the output column.\r\nFor example, if the user request joining a feature named \"careers_job_listTime\" and provides no alias,\r\nthe output FDS will contain a column called \"careers_job_listTime\". However, if the user sets \"featureAlias\" to \"list_time\",\r\nthe column will be named \"list_time\".\r\n\r\nfeature alias can be useful for in a few cases:\r\n - If the user prefers to use a name different than the feathr name in their model,\r\nthey can use an alias to control the name of the column in the output FDS.\r\n - Sometimes, the training datas needs to have two features that are from the same feathr feature.\r\nFor example, if we are modeing the problem of the probability of a member A (viewer) seeing the profile of member B\r\n(viewee) and we want to use the skills of both viewer and viewee as features, we need to join feathr feature\r\n\"member_skills\" of member A with feathr feature \"member_skills\" of member B. That is, the two features are the same\r\nfeature but for different entiity ids). The default behavior of join is to name the output column name using the feathr\r\nfeature name, but in a case like the above case, that would result in two columns with the same name,\r\nwhich is not valid for FDS. In these cases, the user has to provide an alias for at least one of these joined features.\r\nFor example, the user can use featureAliases such as \"viewer_skills\" and \"viewee_skills\".\r\nIn these cases, featureAliases becomes mandatory.*/featureAlias:optional string/**dateRange is used in Time-based joins, which refers to the situation when one or multiple days of input data needs\r\nto be used for training.\r\nOne of the common use cases where this is used, is in training with some time-insensitive features, or\r\ntraining pipeline that always use the full day data, one day before running (since there is only partial data for today).\r\nThe time for the input featurized dataset can be set using this field.\r\nHourly data is not allowed in this case.\r\n\r\nFor example,\r\na. startDate: \"20200522\", endDate: \"20200525\" implies this feature should be joined with the input data starting from\r\n22nd May 2020 to 25th May, 2020 with both dates included.\r\nWe only support yyyyMMdd format for this. In future, if there is a request, we can\r\nadd support for other date time formats as well.\r\n\r\nb. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020\r\ntill 11/04/2020 willl be joined.\r\n\r\nP.S - This is different from the timeRange used in settings as the settings startTime is applicable for the entire input data,\r\nwhile this a feature level setting. Also, we do not support hourly time here.*/dateRange:optional union[absoluteDateRange:/**The absolute date range with start and end date being required fields.\r\nIt accepts a start date and an end date which should be specifiied using the [[Date.pdl]] class.\r\nabsoluteDateRange: {\r\n startDate: Date(day=1, month=1, year=2020)\r\n endDate: Date(day=3, month=1, year=2020)\r\n }\r\n In this case, the endDate > startDate.*/record AbsoluteDateRange{/**start date of the date range, with the start date included in the range.*/startDate:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}/**end date of the date range, with the end date included in the range.*/endDate:Date}relativeDateRange:/**The date range represented relative to the current date. It uses the current system date as the reference and can be used to\r\nexpress a range of dates with respect to the current date.\r\nExample, - If current date is 01/01/2020, window is 3, and offset 1 (unit is number of days)\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nIf dateOffset is not specified, it defaults to 0.\r\nrelativeDateRange: RelativeDateRange(numDays=2, dateOffset=1)\r\nrelativeDateRange: RelativeDateRange(numDays=5)*/record RelativeDateRange{/**Represents a length of time.\r\nnumDays is the window from the reference date to look back to obtain a dateRange.\r\nFor example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020\r\ntill 11/05/2020.*/@validate.positive={}numDays:long/**Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date\r\nwill be 4 days ago from today.*/dateOffset:long=0}]/**The override time delay parameter which will override the global simulate time delay specified in the settings section for\r\nthe particular feature.\r\nThis parameter is only applicable when the simulate time delay is set in the settings section\r\nFor example, let us say the global simulate delay was 5d, and the overrideTimeDelay is set to 3d.\r\nThen, for this specificc feature, a simulate delay of 3d will be applied.*/overrideTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}}}]", SchemaFormatType.PDL)); - - public JoiningFeatureArray() { - this(new DataList()); - } - - public JoiningFeatureArray(int initialCapacity) { - this(new DataList(initialCapacity)); - } - - public JoiningFeatureArray(Collection c) { - this(new DataList(c.size())); - addAll(c); - } - - public JoiningFeatureArray(DataList data) { - super(data, SCHEMA, JoiningFeature.class); - } - - public JoiningFeatureArray(JoiningFeature first, JoiningFeature... rest) { - this(new DataList((rest.length + 1))); - add(first); - addAll(Arrays.asList(rest)); - } - - public static ArrayDataSchema dataSchema() { - return SCHEMA; - } - - public static JoiningFeatureArray.ProjectionMask createMask() { - return new JoiningFeatureArray.ProjectionMask(); - } - - @Override - public JoiningFeatureArray clone() - throws CloneNotSupportedException - { - JoiningFeatureArray __clone = ((JoiningFeatureArray) super.clone()); - return __clone; - } - - @Override - public JoiningFeatureArray copy() - throws CloneNotSupportedException - { - JoiningFeatureArray __copy = ((JoiningFeatureArray) super.copy()); - return __copy; - } - - @Override - protected JoiningFeature coerceOutput(Object object) - throws TemplateOutputCastException - { - assert(object != null); - return ((object == null)?null:new JoiningFeature(DataTemplateUtil.castOrThrow(object, DataMap.class))); - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public com.linkedin.feathr.config.join.JoiningFeature.Fields items() { - return new com.linkedin.feathr.config.join.JoiningFeature.Fields(getPathComponents(), PathSpec.WILDCARD); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.JoiningFeature.ProjectionMask _itemsMask; - - ProjectionMask() { - super(4); - } - - public JoiningFeatureArray.ProjectionMask withItems(Function nestedMask) { - _itemsMask = nestedMask.apply(((_itemsMask == null)?JoiningFeature.createMask():_itemsMask)); - getDataMap().put("$*", _itemsMask.getDataMap()); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeDateRange.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeDateRange.java deleted file mode 100644 index 395d50bd9..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeDateRange.java +++ /dev/null @@ -1,443 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * The date range represented relative to the current date. It uses the current system date as the reference and can be used to - * express a range of dates with respect to the current date. - * Example, - If current date is 01/01/2020, window is 3, and offset 1 (unit is number of days) - * then this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019. - * - * If dateOffset is not specified, it defaults to 0. - * relativeDateRange: RelativeDateRange(numDays=2, dateOffset=1) - * relativeDateRange: RelativeDateRange(numDays=5) - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\RelativeDateRange.pdl.") -public class RelativeDateRange - extends RecordTemplate -{ - - private final static RelativeDateRange.Fields _fields = new RelativeDateRange.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The date range represented relative to the current date. It uses the current system date as the reference and can be used to\r\nexpress a range of dates with respect to the current date.\r\nExample, - If current date is 01/01/2020, window is 3, and offset 1 (unit is number of days)\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nIf dateOffset is not specified, it defaults to 0.\r\nrelativeDateRange: RelativeDateRange(numDays=2, dateOffset=1)\r\nrelativeDateRange: RelativeDateRange(numDays=5)*/record RelativeDateRange{/**Represents a length of time.\r\nnumDays is the window from the reference date to look back to obtain a dateRange.\r\nFor example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020\r\ntill 11/05/2020.*/@validate.positive={}numDays:long/**Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date\r\nwill be 4 days ago from today.*/dateOffset:long=0}", SchemaFormatType.PDL)); - private Long _numDaysField = null; - private Long _dateOffsetField = null; - private RelativeDateRange.ChangeListener __changeListener = new RelativeDateRange.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_NumDays = SCHEMA.getField("numDays"); - private final static RecordDataSchema.Field FIELD_DateOffset = SCHEMA.getField("dateOffset"); - private final static Long DEFAULT_DateOffset; - - static { - DEFAULT_DateOffset = DataTemplateUtil.coerceLongOutput(FIELD_DateOffset.getDefault()); - } - - public RelativeDateRange() { - super(new DataMap(3, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public RelativeDateRange(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static RelativeDateRange.Fields fields() { - return _fields; - } - - public static RelativeDateRange.ProjectionMask createMask() { - return new RelativeDateRange.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for numDays - * - * @see RelativeDateRange.Fields#numDays - */ - public boolean hasNumDays() { - if (_numDaysField!= null) { - return true; - } - return super._map.containsKey("numDays"); - } - - /** - * Remover for numDays - * - * @see RelativeDateRange.Fields#numDays - */ - public void removeNumDays() { - super._map.remove("numDays"); - } - - /** - * Getter for numDays - * - * @see RelativeDateRange.Fields#numDays - */ - public Long getNumDays(GetMode mode) { - switch (mode) { - case STRICT: - return getNumDays(); - case DEFAULT: - case NULL: - if (_numDaysField!= null) { - return _numDaysField; - } else { - Object __rawValue = super._map.get("numDays"); - _numDaysField = DataTemplateUtil.coerceLongOutput(__rawValue); - return _numDaysField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for numDays - * - * @return - * Required field. Could be null for partial record. - * @see RelativeDateRange.Fields#numDays - */ - @Nonnull - public Long getNumDays() { - if (_numDaysField!= null) { - return _numDaysField; - } else { - Object __rawValue = super._map.get("numDays"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("numDays"); - } - _numDaysField = DataTemplateUtil.coerceLongOutput(__rawValue); - return _numDaysField; - } - } - - /** - * Setter for numDays - * - * @see RelativeDateRange.Fields#numDays - */ - public RelativeDateRange setNumDays(Long value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setNumDays(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field numDays of com.linkedin.feathr.config.join.RelativeDateRange"); - } else { - CheckedUtil.putWithoutChecking(super._map, "numDays", DataTemplateUtil.coerceLongInput(value)); - _numDaysField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeNumDays(); - } else { - CheckedUtil.putWithoutChecking(super._map, "numDays", DataTemplateUtil.coerceLongInput(value)); - _numDaysField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "numDays", DataTemplateUtil.coerceLongInput(value)); - _numDaysField = value; - } - break; - } - return this; - } - - /** - * Setter for numDays - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see RelativeDateRange.Fields#numDays - */ - public RelativeDateRange setNumDays( - @Nonnull - Long value) { - if (value == null) { - throw new NullPointerException("Cannot set field numDays of com.linkedin.feathr.config.join.RelativeDateRange to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "numDays", DataTemplateUtil.coerceLongInput(value)); - _numDaysField = value; - } - return this; - } - - /** - * Setter for numDays - * - * @see RelativeDateRange.Fields#numDays - */ - public RelativeDateRange setNumDays(long value) { - CheckedUtil.putWithoutChecking(super._map, "numDays", DataTemplateUtil.coerceLongInput(value)); - _numDaysField = value; - return this; - } - - /** - * Existence checker for dateOffset - * - * @see RelativeDateRange.Fields#dateOffset - */ - public boolean hasDateOffset() { - if (_dateOffsetField!= null) { - return true; - } - return super._map.containsKey("dateOffset"); - } - - /** - * Remover for dateOffset - * - * @see RelativeDateRange.Fields#dateOffset - */ - public void removeDateOffset() { - super._map.remove("dateOffset"); - } - - /** - * Getter for dateOffset - * - * @see RelativeDateRange.Fields#dateOffset - */ - public Long getDateOffset(GetMode mode) { - switch (mode) { - case STRICT: - case DEFAULT: - return getDateOffset(); - case NULL: - if (_dateOffsetField!= null) { - return _dateOffsetField; - } else { - Object __rawValue = super._map.get("dateOffset"); - _dateOffsetField = DataTemplateUtil.coerceLongOutput(__rawValue); - return _dateOffsetField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for dateOffset - * - * @return - * Required field. Could be null for partial record. - * @see RelativeDateRange.Fields#dateOffset - */ - @Nonnull - public Long getDateOffset() { - if (_dateOffsetField!= null) { - return _dateOffsetField; - } else { - Object __rawValue = super._map.get("dateOffset"); - if (__rawValue == null) { - return DEFAULT_DateOffset; - } - _dateOffsetField = DataTemplateUtil.coerceLongOutput(__rawValue); - return _dateOffsetField; - } - } - - /** - * Setter for dateOffset - * - * @see RelativeDateRange.Fields#dateOffset - */ - public RelativeDateRange setDateOffset(Long value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setDateOffset(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field dateOffset of com.linkedin.feathr.config.join.RelativeDateRange"); - } else { - CheckedUtil.putWithoutChecking(super._map, "dateOffset", DataTemplateUtil.coerceLongInput(value)); - _dateOffsetField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeDateOffset(); - } else { - CheckedUtil.putWithoutChecking(super._map, "dateOffset", DataTemplateUtil.coerceLongInput(value)); - _dateOffsetField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "dateOffset", DataTemplateUtil.coerceLongInput(value)); - _dateOffsetField = value; - } - break; - } - return this; - } - - /** - * Setter for dateOffset - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see RelativeDateRange.Fields#dateOffset - */ - public RelativeDateRange setDateOffset( - @Nonnull - Long value) { - if (value == null) { - throw new NullPointerException("Cannot set field dateOffset of com.linkedin.feathr.config.join.RelativeDateRange to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "dateOffset", DataTemplateUtil.coerceLongInput(value)); - _dateOffsetField = value; - } - return this; - } - - /** - * Setter for dateOffset - * - * @see RelativeDateRange.Fields#dateOffset - */ - public RelativeDateRange setDateOffset(long value) { - CheckedUtil.putWithoutChecking(super._map, "dateOffset", DataTemplateUtil.coerceLongInput(value)); - _dateOffsetField = value; - return this; - } - - @Override - public RelativeDateRange clone() - throws CloneNotSupportedException - { - RelativeDateRange __clone = ((RelativeDateRange) super.clone()); - __clone.__changeListener = new RelativeDateRange.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public RelativeDateRange copy() - throws CloneNotSupportedException - { - RelativeDateRange __copy = ((RelativeDateRange) super.copy()); - __copy._numDaysField = null; - __copy._dateOffsetField = null; - __copy.__changeListener = new RelativeDateRange.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final RelativeDateRange __objectRef; - - private ChangeListener(RelativeDateRange reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "numDays": - __objectRef._numDaysField = null; - break; - case "dateOffset": - __objectRef._dateOffsetField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Represents a length of time. - * numDays is the window from the reference date to look back to obtain a dateRange. - * For example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020 - * till 11/05/2020. - * - */ - public PathSpec numDays() { - return new PathSpec(getPathComponents(), "numDays"); - } - - /** - * Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date - * will be 4 days ago from today. - * - */ - public PathSpec dateOffset() { - return new PathSpec(getPathComponents(), "dateOffset"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(3); - } - - /** - * Represents a length of time. - * numDays is the window from the reference date to look back to obtain a dateRange. - * For example, numDays - 5 implies, if reference date is 11/09/2020, then numDays will range from 11/09/2020 - * till 11/05/2020. - * - */ - public RelativeDateRange.ProjectionMask withNumDays() { - getDataMap().put("numDays", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Number of days to backdate from current date, to obtain the reference date. For example, if dateOffset is 4, then reference date - * will be 4 days ago from today. - * - */ - public RelativeDateRange.ProjectionMask withDateOffset() { - getDataMap().put("dateOffset", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeTimeRange.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeTimeRange.java deleted file mode 100644 index 30d773502..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/RelativeTimeRange.java +++ /dev/null @@ -1,453 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * The time range represented relative to the current timestamp. It uses the current system time as the reference and can be used to - * express a range of times with respect to the current time. - * Example, - If current time is 01/01/2020, window is 3 days, and offset is 1 day (unit can be day or hour). - * then this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019. - * - * relativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit="DAY"), offset=TimeOffset(length=1, unit="Day")) - * relativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit="HOUR")) - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\RelativeTimeRange.pdl.") -public class RelativeTimeRange - extends RecordTemplate -{ - - private final static RelativeTimeRange.Fields _fields = new RelativeTimeRange.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The time range represented relative to the current timestamp. It uses the current system time as the reference and can be used to\r\nexpress a range of times with respect to the current time.\r\nExample, - If current time is 01/01/2020, window is 3 days, and offset is 1 day (unit can be day or hour).\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"DAY\"), offset=TimeOffset(length=1, unit=\"Day\"))\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"HOUR\"))*/record RelativeTimeRange{/**Window is the number of time units from the reference time units to look back to obtain the timeRange.\r\nFor example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020\r\ntill 11/05/2020 (both days included).\r\nwindow >= 1 TimeUnit*/window:/**Represents a length of time along with the corresponding time unit (DAY, HOUR).*/record TimeWindow{/**Amount of the duration in TimeUnits. Can be greater or equal to 1.*/@validate.positive,length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}/**Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time.\r\nFor example, if dateOffset is 4, and window is 2 days, then reference time\r\nwill be 4 days ago from today.\r\nExample - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020.\r\nThis will always take the window's timeUnits.*/@validate.integerRange.min=0,offset:long=0}", SchemaFormatType.PDL)); - private TimeWindow _windowField = null; - private Long _offsetField = null; - private RelativeTimeRange.ChangeListener __changeListener = new RelativeTimeRange.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Window = SCHEMA.getField("window"); - private final static RecordDataSchema.Field FIELD_Offset = SCHEMA.getField("offset"); - private final static Long DEFAULT_Offset; - - static { - DEFAULT_Offset = DataTemplateUtil.coerceLongOutput(FIELD_Offset.getDefault()); - } - - public RelativeTimeRange() { - super(new DataMap(3, 0.75F), SCHEMA, 2); - addChangeListener(__changeListener); - } - - public RelativeTimeRange(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static RelativeTimeRange.Fields fields() { - return _fields; - } - - public static RelativeTimeRange.ProjectionMask createMask() { - return new RelativeTimeRange.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for window - * - * @see RelativeTimeRange.Fields#window - */ - public boolean hasWindow() { - if (_windowField!= null) { - return true; - } - return super._map.containsKey("window"); - } - - /** - * Remover for window - * - * @see RelativeTimeRange.Fields#window - */ - public void removeWindow() { - super._map.remove("window"); - } - - /** - * Getter for window - * - * @see RelativeTimeRange.Fields#window - */ - public TimeWindow getWindow(GetMode mode) { - switch (mode) { - case STRICT: - return getWindow(); - case DEFAULT: - case NULL: - if (_windowField!= null) { - return _windowField; - } else { - Object __rawValue = super._map.get("window"); - _windowField = ((__rawValue == null)?null:new TimeWindow(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _windowField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for window - * - * @return - * Required field. Could be null for partial record. - * @see RelativeTimeRange.Fields#window - */ - @Nonnull - public TimeWindow getWindow() { - if (_windowField!= null) { - return _windowField; - } else { - Object __rawValue = super._map.get("window"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("window"); - } - _windowField = ((__rawValue == null)?null:new TimeWindow(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _windowField; - } - } - - /** - * Setter for window - * - * @see RelativeTimeRange.Fields#window - */ - public RelativeTimeRange setWindow(TimeWindow value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setWindow(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field window of com.linkedin.feathr.config.join.RelativeTimeRange"); - } else { - CheckedUtil.putWithoutChecking(super._map, "window", value.data()); - _windowField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeWindow(); - } else { - CheckedUtil.putWithoutChecking(super._map, "window", value.data()); - _windowField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "window", value.data()); - _windowField = value; - } - break; - } - return this; - } - - /** - * Setter for window - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see RelativeTimeRange.Fields#window - */ - public RelativeTimeRange setWindow( - @Nonnull - TimeWindow value) { - if (value == null) { - throw new NullPointerException("Cannot set field window of com.linkedin.feathr.config.join.RelativeTimeRange to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "window", value.data()); - _windowField = value; - } - return this; - } - - /** - * Existence checker for offset - * - * @see RelativeTimeRange.Fields#offset - */ - public boolean hasOffset() { - if (_offsetField!= null) { - return true; - } - return super._map.containsKey("offset"); - } - - /** - * Remover for offset - * - * @see RelativeTimeRange.Fields#offset - */ - public void removeOffset() { - super._map.remove("offset"); - } - - /** - * Getter for offset - * - * @see RelativeTimeRange.Fields#offset - */ - public Long getOffset(GetMode mode) { - switch (mode) { - case STRICT: - case DEFAULT: - return getOffset(); - case NULL: - if (_offsetField!= null) { - return _offsetField; - } else { - Object __rawValue = super._map.get("offset"); - _offsetField = DataTemplateUtil.coerceLongOutput(__rawValue); - return _offsetField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for offset - * - * @return - * Required field. Could be null for partial record. - * @see RelativeTimeRange.Fields#offset - */ - @Nonnull - public Long getOffset() { - if (_offsetField!= null) { - return _offsetField; - } else { - Object __rawValue = super._map.get("offset"); - if (__rawValue == null) { - return DEFAULT_Offset; - } - _offsetField = DataTemplateUtil.coerceLongOutput(__rawValue); - return _offsetField; - } - } - - /** - * Setter for offset - * - * @see RelativeTimeRange.Fields#offset - */ - public RelativeTimeRange setOffset(Long value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setOffset(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field offset of com.linkedin.feathr.config.join.RelativeTimeRange"); - } else { - CheckedUtil.putWithoutChecking(super._map, "offset", DataTemplateUtil.coerceLongInput(value)); - _offsetField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeOffset(); - } else { - CheckedUtil.putWithoutChecking(super._map, "offset", DataTemplateUtil.coerceLongInput(value)); - _offsetField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "offset", DataTemplateUtil.coerceLongInput(value)); - _offsetField = value; - } - break; - } - return this; - } - - /** - * Setter for offset - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see RelativeTimeRange.Fields#offset - */ - public RelativeTimeRange setOffset( - @Nonnull - Long value) { - if (value == null) { - throw new NullPointerException("Cannot set field offset of com.linkedin.feathr.config.join.RelativeTimeRange to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "offset", DataTemplateUtil.coerceLongInput(value)); - _offsetField = value; - } - return this; - } - - /** - * Setter for offset - * - * @see RelativeTimeRange.Fields#offset - */ - public RelativeTimeRange setOffset(long value) { - CheckedUtil.putWithoutChecking(super._map, "offset", DataTemplateUtil.coerceLongInput(value)); - _offsetField = value; - return this; - } - - @Override - public RelativeTimeRange clone() - throws CloneNotSupportedException - { - RelativeTimeRange __clone = ((RelativeTimeRange) super.clone()); - __clone.__changeListener = new RelativeTimeRange.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public RelativeTimeRange copy() - throws CloneNotSupportedException - { - RelativeTimeRange __copy = ((RelativeTimeRange) super.copy()); - __copy._offsetField = null; - __copy._windowField = null; - __copy.__changeListener = new RelativeTimeRange.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final RelativeTimeRange __objectRef; - - private ChangeListener(RelativeTimeRange reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "offset": - __objectRef._offsetField = null; - break; - case "window": - __objectRef._windowField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Window is the number of time units from the reference time units to look back to obtain the timeRange. - * For example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020 - * till 11/05/2020 (both days included). - * window >= 1 TimeUnit - * - */ - public com.linkedin.feathr.config.join.TimeWindow.Fields window() { - return new com.linkedin.feathr.config.join.TimeWindow.Fields(getPathComponents(), "window"); - } - - /** - * Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time. - * For example, if dateOffset is 4, and window is 2 days, then reference time - * will be 4 days ago from today. - * Example - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020. - * This will always take the window's timeUnits. - * - */ - public PathSpec offset() { - return new PathSpec(getPathComponents(), "offset"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.TimeWindow.ProjectionMask _windowMask; - - ProjectionMask() { - super(3); - } - - /** - * Window is the number of time units from the reference time units to look back to obtain the timeRange. - * For example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020 - * till 11/05/2020 (both days included). - * window >= 1 TimeUnit - * - */ - public RelativeTimeRange.ProjectionMask withWindow(Function nestedMask) { - _windowMask = nestedMask.apply(((_windowMask == null)?TimeWindow.createMask():_windowMask)); - getDataMap().put("window", _windowMask.getDataMap()); - return this; - } - - /** - * Window is the number of time units from the reference time units to look back to obtain the timeRange. - * For example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020 - * till 11/05/2020 (both days included). - * window >= 1 TimeUnit - * - */ - public RelativeTimeRange.ProjectionMask withWindow() { - _windowMask = null; - getDataMap().put("window", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time. - * For example, if dateOffset is 4, and window is 2 days, then reference time - * will be 4 days ago from today. - * Example - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020. - * This will always take the window's timeUnits. - * - */ - public RelativeTimeRange.ProjectionMask withOffset() { - getDataMap().put("offset", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Settings.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Settings.java deleted file mode 100644 index 0b259d586..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/Settings.java +++ /dev/null @@ -1,400 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.SetMode; - - -/** - * The settings section contains all the config parameters required for the joining of input dataset with the - * feature data. As of now, we have only time related parameters, but in future this can be expanded. - * This section has configs related to:- - * a. How do I load the input dataset if it is time sensitive? - * b. How do I specify the join parameters for input dataset? - * For more details - https://docs.google.com/document/d/1C6u2CKWSmOmHDQEL8Ovm5V5ZZFKhC_HdxVxU9D1F9lg/edit# - * settings: { - * inputDataTimeSettings: { - * absoluteTimeRange: { - * startTime: 20200809 - * endTime: 20200810 - * timeFormat: yyyyMMdd - * } - * } - * joinTimeSettings: { - * useLatestFeatureData: true - * } - * } - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\Settings.pdl.") -public class Settings - extends RecordTemplate -{ - - private final static Settings.Fields _fields = new Settings.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The settings section contains all the config parameters required for the joining of input dataset with the\r\nfeature data. As of now, we have only time related parameters, but in future this can be expanded.\r\nThis section has configs related to:-\r\na. How do I load the input dataset if it is time sensitive?\r\nb. How do I specify the join parameters for input dataset?\r\nFor more details - https://docs.google.com/document/d/1C6u2CKWSmOmHDQEL8Ovm5V5ZZFKhC_HdxVxU9D1F9lg/edit#\r\nsettings: {\r\n inputDataTimeSettings: {\r\n absoluteTimeRange: {\r\n startTime: 20200809\r\n endTime: 20200810\r\n timeFormat: yyyyMMdd\r\n }\r\n }\r\n joinTimeSettings: {\r\n useLatestFeatureData: true\r\n }\r\n}*/record Settings{/**Config parameters related to loading of the time sensitive input data. Contains parameters related to restricting the\r\nsize of the input data with respect to the timestamp column.*/inputDataTimeSettings:optional/**The data time settings pertaining to how much of the input dataset is to be loaded from the timestamp column. This is a way in which\r\nthe input data can be restricted to allow only a fixed interval of dates to be joined with the feature data. This restriction\r\nwill apply on the timestamp column of the input data.\r\ninputDataTimeSettings: {\r\n absoluteTimeRange: {\r\n startTime: Date(year=2020, month=8, day=8)\r\n endTime: Date(year=2020, month=8, day=10)\r\n }\r\n (or)\r\n relativeTimeRange: {\r\n offset: TimeOffset(length=1, unit=\"DAY\")\r\n window: TimeWindow(length=1, unit=\"DAY\")\r\n }\r\n}*/record InputDataTimeSettings{/**Union of [[AbsoluteTimeRange]] and [[RelativeTimeRange]].\r\nIt indicates the range of input data which is to be loaded. This field generally refers to how much of the input\r\ndata should be restricted using the time in the timestamp column.\r\n\r\nFor example,\r\na. startDate: \"20200522\", endDate: \"20200525\" implies this feature should be joined with the input data starting from\r\n22nd May 2020 to 25th May, 2020 with both dates included.\r\nWe only support yyyyMMdd format for this. In future, if there is a request, we can\r\nadd support for other date time formats as well.\r\n\r\nb. numDays - 5d implies, offset - 1d, if today's date is 11/09/2020, then the input data ranging from 11/08/2020\r\ntill 11/04/2020 willl be joined.*/timeRange:union[absoluteTimeRange:/**The absolute time range with start and end time being required fields.\r\nIt accepts a start time and an end time which should be specifiied using the [[Date.pdl]] or the [[HourTime.pdl]] class.\r\nThis model can be used to represent time range in daily or hourly interval.\r\nabsoluteTimeRange: {\r\n startTime: TimeHour(day=1, month=1, year=2020, hour=13)\r\n endTime: TimeHour(day=3, month=1, year=2020, hour=2)\r\n }\r\n(or)\r\nabsoluteTimeRange: {\r\n startTime: Date(day=1, month=1, year=2020)\r\n endTime: Date(day=3, month=1, year=2020)\r\n }\r\nendTime and startTime should always have the same granularity, ie - Daily or Hourly.\r\nendTme > startTime*/record AbsoluteTimeRange{/**start time of the date range, in daily or hourly format with the start date included in the range.*/startTime:union[date:/**Represents a date in a calendar year including day, year and month*/record Date{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int}hourTime:/**Time with hourly granularity*/record HourTime{/**day*/@validate.integerRange={\"max\":31,\"min\":1}day:int/**month*/@validate.integerRange={\"max\":12,\"min\":1}month:int/**year*/@validate.integerRange={\"max\":2099,\"min\":1970}year:int/**hour*/@validate.integerRange={\"max\":23,\"min\":0}hour:int}]/**end date of the date range, in daily or hourly format with the end date included in the range.*/endTime:union[date:Date,hourTime:HourTime]}relativeTimeRange:/**The time range represented relative to the current timestamp. It uses the current system time as the reference and can be used to\r\nexpress a range of times with respect to the current time.\r\nExample, - If current time is 01/01/2020, window is 3 days, and offset is 1 day (unit can be day or hour).\r\nthen this corresponds to the following 3 days, ie- starting from (current date - offset), ie - 12/31/2019, 12/30/2019 and 12/29/2019.\r\n\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"DAY\"), offset=TimeOffset(length=1, unit=\"Day\"))\r\nrelativeTimeRange: RelativeTimeRange(window=TimeWindow(length=2, unit=\"HOUR\"))*/record RelativeTimeRange{/**Window is the number of time units from the reference time units to look back to obtain the timeRange.\r\nFor example, window - 5days implies, if reference date is 11/09/2020, then range will be from 11/09/2020\r\ntill 11/05/2020 (both days included).\r\nwindow >= 1 TimeUnit*/window:/**Represents a length of time along with the corresponding time unit (DAY, HOUR).*/record TimeWindow{/**Amount of the duration in TimeUnits. Can be greater or equal to 1.*/@validate.positive,length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}/**Number of time units (corresponding to window's timeUnits) to backdate from current time, to obtain the reference time.\r\nFor example, if dateOffset is 4, and window is 2 days, then reference time\r\nwill be 4 days ago from today.\r\nExample - if today's date is 11th Dec, 2020 and offset is 4 days - Reference time will be 7th Dec, 2020.\r\nThis will always take the window's timeUnits.*/@validate.integerRange.min=0,offset:long=0}]}/**This contains all the parameters required to join the time sensitive input data with the feature data.*/joinTimeSettings:optional/**JoinTimeSettings contains all the parameters required to join the time sensitive input data with the feature data.\r\nThe input data can be time sensitive in two ways:-\r\na. Have a timestamp column\r\nb. Always join with the latest available feature data. In this case, we do not require a timestamp column.\r\nc. The file path is time-partition and the path time is used for the join\r\n(Todo - Add useTimePartitionPattern field in this section)\r\nIn this section, the user needs to let feathr know which of the above properties is to be used for the join.*/typeref JoinTimeSettings=union[useLatestJoinTimeSettings:/**Settings needed when the input data is to be joined with the latest available feature data.\r\njoinTimeSettings: {\r\n useLatestFeatureData: true\r\n}*/record UseLatestJoinTimeSettings{/**Boolean value, if set to true, indicates that the latest available feature data is to be used for joining.\r\nWhen useLatestFeatureData is set, there should be no other time-based parameters.*/useLatestFeatureData:boolean=true}timestampColJoinTimeSettings:/**Settings needed when the input data has a timestamp which should be used for the join.\r\njoinTimeSettings: {\r\n timestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }\r\n simulateTimeDelay: 1d\r\n}*/record TimestampColJoinTimeSettings{/**The timestamp column name and timeformat which should be used for joining with the feature data.\r\nRefer to [[TimestampColumn]].\r\nExample, TimestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }*/timestampColumn:/**Timestamp column of the input featureiized dataset, which is to be used for the join.\r\ntimestampColumn: {\r\n def: timestamp\r\n format: yyyyMMdd\r\n }*/record TimestampColumn{/**The definiton of the timestamp column, which can be a sql expression involving the timestamp column\r\nor just the column name\r\nExample:- definition: timestamp, timestamp + 10000000.*/definition:union[columnName:string,sparkSqlExpression:/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}]/**Format of the timestamp column. Must confer to java's timestampFormatter or can be\r\nepoch or epoch_millis.\r\nExample:- epoch, epoch_millis, yyyy/MM/dd*/format:/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string}/**An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted\r\nfrom the input data timestamp while joining with the feature data.\r\nWe do support negative time delays.*/simulateTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:TimeUnit}}]}", SchemaFormatType.PDL)); - private InputDataTimeSettings _inputDataTimeSettingsField = null; - private JoinTimeSettings _joinTimeSettingsField = null; - private Settings.ChangeListener __changeListener = new Settings.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_InputDataTimeSettings = SCHEMA.getField("inputDataTimeSettings"); - private final static RecordDataSchema.Field FIELD_JoinTimeSettings = SCHEMA.getField("joinTimeSettings"); - - public Settings() { - super(new DataMap(3, 0.75F), SCHEMA, 3); - addChangeListener(__changeListener); - } - - public Settings(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static Settings.Fields fields() { - return _fields; - } - - public static Settings.ProjectionMask createMask() { - return new Settings.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for inputDataTimeSettings - * - * @see Settings.Fields#inputDataTimeSettings - */ - public boolean hasInputDataTimeSettings() { - if (_inputDataTimeSettingsField!= null) { - return true; - } - return super._map.containsKey("inputDataTimeSettings"); - } - - /** - * Remover for inputDataTimeSettings - * - * @see Settings.Fields#inputDataTimeSettings - */ - public void removeInputDataTimeSettings() { - super._map.remove("inputDataTimeSettings"); - } - - /** - * Getter for inputDataTimeSettings - * - * @see Settings.Fields#inputDataTimeSettings - */ - public InputDataTimeSettings getInputDataTimeSettings(GetMode mode) { - return getInputDataTimeSettings(); - } - - /** - * Getter for inputDataTimeSettings - * - * @return - * Optional field. Always check for null. - * @see Settings.Fields#inputDataTimeSettings - */ - @Nullable - public InputDataTimeSettings getInputDataTimeSettings() { - if (_inputDataTimeSettingsField!= null) { - return _inputDataTimeSettingsField; - } else { - Object __rawValue = super._map.get("inputDataTimeSettings"); - _inputDataTimeSettingsField = ((__rawValue == null)?null:new InputDataTimeSettings(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _inputDataTimeSettingsField; - } - } - - /** - * Setter for inputDataTimeSettings - * - * @see Settings.Fields#inputDataTimeSettings - */ - public Settings setInputDataTimeSettings(InputDataTimeSettings value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setInputDataTimeSettings(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeInputDataTimeSettings(); - } else { - CheckedUtil.putWithoutChecking(super._map, "inputDataTimeSettings", value.data()); - _inputDataTimeSettingsField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "inputDataTimeSettings", value.data()); - _inputDataTimeSettingsField = value; - } - break; - } - return this; - } - - /** - * Setter for inputDataTimeSettings - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Settings.Fields#inputDataTimeSettings - */ - public Settings setInputDataTimeSettings( - @Nonnull - InputDataTimeSettings value) { - if (value == null) { - throw new NullPointerException("Cannot set field inputDataTimeSettings of com.linkedin.feathr.config.join.Settings to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "inputDataTimeSettings", value.data()); - _inputDataTimeSettingsField = value; - } - return this; - } - - /** - * Existence checker for joinTimeSettings - * - * @see Settings.Fields#joinTimeSettings - */ - public boolean hasJoinTimeSettings() { - if (_joinTimeSettingsField!= null) { - return true; - } - return super._map.containsKey("joinTimeSettings"); - } - - /** - * Remover for joinTimeSettings - * - * @see Settings.Fields#joinTimeSettings - */ - public void removeJoinTimeSettings() { - super._map.remove("joinTimeSettings"); - } - - /** - * Getter for joinTimeSettings - * - * @see Settings.Fields#joinTimeSettings - */ - public JoinTimeSettings getJoinTimeSettings(GetMode mode) { - return getJoinTimeSettings(); - } - - /** - * Getter for joinTimeSettings - * - * @return - * Optional field. Always check for null. - * @see Settings.Fields#joinTimeSettings - */ - @Nullable - public JoinTimeSettings getJoinTimeSettings() { - if (_joinTimeSettingsField!= null) { - return _joinTimeSettingsField; - } else { - Object __rawValue = super._map.get("joinTimeSettings"); - _joinTimeSettingsField = ((__rawValue == null)?null:new JoinTimeSettings(__rawValue)); - return _joinTimeSettingsField; - } - } - - /** - * Setter for joinTimeSettings - * - * @see Settings.Fields#joinTimeSettings - */ - public Settings setJoinTimeSettings(JoinTimeSettings value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setJoinTimeSettings(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeJoinTimeSettings(); - } else { - CheckedUtil.putWithoutChecking(super._map, "joinTimeSettings", value.data()); - _joinTimeSettingsField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "joinTimeSettings", value.data()); - _joinTimeSettingsField = value; - } - break; - } - return this; - } - - /** - * Setter for joinTimeSettings - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see Settings.Fields#joinTimeSettings - */ - public Settings setJoinTimeSettings( - @Nonnull - JoinTimeSettings value) { - if (value == null) { - throw new NullPointerException("Cannot set field joinTimeSettings of com.linkedin.feathr.config.join.Settings to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "joinTimeSettings", value.data()); - _joinTimeSettingsField = value; - } - return this; - } - - @Override - public Settings clone() - throws CloneNotSupportedException - { - Settings __clone = ((Settings) super.clone()); - __clone.__changeListener = new Settings.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public Settings copy() - throws CloneNotSupportedException - { - Settings __copy = ((Settings) super.copy()); - __copy._inputDataTimeSettingsField = null; - __copy._joinTimeSettingsField = null; - __copy.__changeListener = new Settings.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final Settings __objectRef; - - private ChangeListener(Settings reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "inputDataTimeSettings": - __objectRef._inputDataTimeSettingsField = null; - break; - case "joinTimeSettings": - __objectRef._joinTimeSettingsField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Config parameters related to loading of the time sensitive input data. Contains parameters related to restricting the - * size of the input data with respect to the timestamp column. - * - */ - public com.linkedin.feathr.config.join.InputDataTimeSettings.Fields inputDataTimeSettings() { - return new com.linkedin.feathr.config.join.InputDataTimeSettings.Fields(getPathComponents(), "inputDataTimeSettings"); - } - - /** - * This contains all the parameters required to join the time sensitive input data with the feature data. - * - */ - public com.linkedin.feathr.config.join.JoinTimeSettings.Fields joinTimeSettings() { - return new com.linkedin.feathr.config.join.JoinTimeSettings.Fields(getPathComponents(), "joinTimeSettings"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.InputDataTimeSettings.ProjectionMask _inputDataTimeSettingsMask; - private com.linkedin.feathr.config.join.JoinTimeSettings.ProjectionMask _joinTimeSettingsMask; - - ProjectionMask() { - super(3); - } - - /** - * Config parameters related to loading of the time sensitive input data. Contains parameters related to restricting the - * size of the input data with respect to the timestamp column. - * - */ - public Settings.ProjectionMask withInputDataTimeSettings(Function nestedMask) { - _inputDataTimeSettingsMask = nestedMask.apply(((_inputDataTimeSettingsMask == null)?InputDataTimeSettings.createMask():_inputDataTimeSettingsMask)); - getDataMap().put("inputDataTimeSettings", _inputDataTimeSettingsMask.getDataMap()); - return this; - } - - /** - * Config parameters related to loading of the time sensitive input data. Contains parameters related to restricting the - * size of the input data with respect to the timestamp column. - * - */ - public Settings.ProjectionMask withInputDataTimeSettings() { - _inputDataTimeSettingsMask = null; - getDataMap().put("inputDataTimeSettings", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * This contains all the parameters required to join the time sensitive input data with the feature data. - * - */ - public Settings.ProjectionMask withJoinTimeSettings(Function nestedMask) { - _joinTimeSettingsMask = nestedMask.apply(((_joinTimeSettingsMask == null)?JoinTimeSettings.createMask():_joinTimeSettingsMask)); - getDataMap().put("joinTimeSettings", _joinTimeSettingsMask.getDataMap()); - return this; - } - - /** - * This contains all the parameters required to join the time sensitive input data with the feature data. - * - */ - public Settings.ProjectionMask withJoinTimeSettings() { - _joinTimeSettingsMask = null; - getDataMap().put("joinTimeSettings", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/SparkSqlExpression.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/SparkSqlExpression.java deleted file mode 100644 index 35aeff228..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/SparkSqlExpression.java +++ /dev/null @@ -1,260 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * An expression in Spark SQL. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\SparkSqlExpression.pdl.") -public class SparkSqlExpression - extends RecordTemplate -{ - - private final static SparkSqlExpression.Fields _fields = new SparkSqlExpression.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}", SchemaFormatType.PDL)); - private String _expressionField = null; - private SparkSqlExpression.ChangeListener __changeListener = new SparkSqlExpression.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Expression = SCHEMA.getField("expression"); - - public SparkSqlExpression() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public SparkSqlExpression(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static SparkSqlExpression.Fields fields() { - return _fields; - } - - public static SparkSqlExpression.ProjectionMask createMask() { - return new SparkSqlExpression.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for expression - * - * @see SparkSqlExpression.Fields#expression - */ - public boolean hasExpression() { - if (_expressionField!= null) { - return true; - } - return super._map.containsKey("expression"); - } - - /** - * Remover for expression - * - * @see SparkSqlExpression.Fields#expression - */ - public void removeExpression() { - super._map.remove("expression"); - } - - /** - * Getter for expression - * - * @see SparkSqlExpression.Fields#expression - */ - public String getExpression(GetMode mode) { - switch (mode) { - case STRICT: - return getExpression(); - case DEFAULT: - case NULL: - if (_expressionField!= null) { - return _expressionField; - } else { - Object __rawValue = super._map.get("expression"); - _expressionField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _expressionField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for expression - * - * @return - * Required field. Could be null for partial record. - * @see SparkSqlExpression.Fields#expression - */ - @Nonnull - public String getExpression() { - if (_expressionField!= null) { - return _expressionField; - } else { - Object __rawValue = super._map.get("expression"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("expression"); - } - _expressionField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _expressionField; - } - } - - /** - * Setter for expression - * - * @see SparkSqlExpression.Fields#expression - */ - public SparkSqlExpression setExpression(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setExpression(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field expression of com.linkedin.feathr.config.join.SparkSqlExpression"); - } else { - CheckedUtil.putWithoutChecking(super._map, "expression", value); - _expressionField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeExpression(); - } else { - CheckedUtil.putWithoutChecking(super._map, "expression", value); - _expressionField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "expression", value); - _expressionField = value; - } - break; - } - return this; - } - - /** - * Setter for expression - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see SparkSqlExpression.Fields#expression - */ - public SparkSqlExpression setExpression( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field expression of com.linkedin.feathr.config.join.SparkSqlExpression to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "expression", value); - _expressionField = value; - } - return this; - } - - @Override - public SparkSqlExpression clone() - throws CloneNotSupportedException - { - SparkSqlExpression __clone = ((SparkSqlExpression) super.clone()); - __clone.__changeListener = new SparkSqlExpression.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public SparkSqlExpression copy() - throws CloneNotSupportedException - { - SparkSqlExpression __copy = ((SparkSqlExpression) super.copy()); - __copy._expressionField = null; - __copy.__changeListener = new SparkSqlExpression.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final SparkSqlExpression __objectRef; - - private ChangeListener(SparkSqlExpression reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "expression": - __objectRef._expressionField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The Spark SQL expression. - * - */ - public PathSpec expression() { - return new PathSpec(getPathComponents(), "expression"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(2); - } - - /** - * The Spark SQL expression. - * - */ - public SparkSqlExpression.ProjectionMask withExpression() { - getDataMap().put("expression", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeFormat.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeFormat.java deleted file mode 100644 index 3ff7d5455..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeFormat.java +++ /dev/null @@ -1,31 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import javax.annotation.Generated; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.TyperefDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.TyperefInfo; - - -/** - * The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have - * the option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimeFormat.pdl.") -public class TimeFormat - extends TyperefInfo -{ - - private final static TyperefDataSchema SCHEMA = ((TyperefDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string", SchemaFormatType.PDL)); - - public TimeFormat() { - super(SCHEMA); - } - - public static TyperefDataSchema dataSchema() { - return SCHEMA; - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeOffset.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeOffset.java deleted file mode 100644 index 4132a962f..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeOffset.java +++ /dev/null @@ -1,414 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can - * be any time in the past also, we do allow a positive or negative offset length. - * offset - 1 day implies the previous from the reference day. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimeOffset.pdl.") -public class TimeOffset - extends RecordTemplate -{ - - private final static TimeOffset.Fields _fields = new TimeOffset.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}", SchemaFormatType.PDL)); - private Long _lengthField = null; - private TimeUnit _unitField = null; - private TimeOffset.ChangeListener __changeListener = new TimeOffset.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Length = SCHEMA.getField("length"); - private final static RecordDataSchema.Field FIELD_Unit = SCHEMA.getField("unit"); - - public TimeOffset() { - super(new DataMap(3, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public TimeOffset(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static TimeOffset.Fields fields() { - return _fields; - } - - public static TimeOffset.ProjectionMask createMask() { - return new TimeOffset.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for length - * - * @see TimeOffset.Fields#length - */ - public boolean hasLength() { - if (_lengthField!= null) { - return true; - } - return super._map.containsKey("length"); - } - - /** - * Remover for length - * - * @see TimeOffset.Fields#length - */ - public void removeLength() { - super._map.remove("length"); - } - - /** - * Getter for length - * - * @see TimeOffset.Fields#length - */ - public Long getLength(GetMode mode) { - switch (mode) { - case STRICT: - return getLength(); - case DEFAULT: - case NULL: - if (_lengthField!= null) { - return _lengthField; - } else { - Object __rawValue = super._map.get("length"); - _lengthField = DataTemplateUtil.coerceLongOutput(__rawValue); - return _lengthField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for length - * - * @return - * Required field. Could be null for partial record. - * @see TimeOffset.Fields#length - */ - @Nonnull - public Long getLength() { - if (_lengthField!= null) { - return _lengthField; - } else { - Object __rawValue = super._map.get("length"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("length"); - } - _lengthField = DataTemplateUtil.coerceLongOutput(__rawValue); - return _lengthField; - } - } - - /** - * Setter for length - * - * @see TimeOffset.Fields#length - */ - public TimeOffset setLength(Long value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setLength(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field length of com.linkedin.feathr.config.join.TimeOffset"); - } else { - CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); - _lengthField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeLength(); - } else { - CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); - _lengthField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); - _lengthField = value; - } - break; - } - return this; - } - - /** - * Setter for length - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TimeOffset.Fields#length - */ - public TimeOffset setLength( - @Nonnull - Long value) { - if (value == null) { - throw new NullPointerException("Cannot set field length of com.linkedin.feathr.config.join.TimeOffset to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); - _lengthField = value; - } - return this; - } - - /** - * Setter for length - * - * @see TimeOffset.Fields#length - */ - public TimeOffset setLength(long value) { - CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); - _lengthField = value; - return this; - } - - /** - * Existence checker for unit - * - * @see TimeOffset.Fields#unit - */ - public boolean hasUnit() { - if (_unitField!= null) { - return true; - } - return super._map.containsKey("unit"); - } - - /** - * Remover for unit - * - * @see TimeOffset.Fields#unit - */ - public void removeUnit() { - super._map.remove("unit"); - } - - /** - * Getter for unit - * - * @see TimeOffset.Fields#unit - */ - public TimeUnit getUnit(GetMode mode) { - switch (mode) { - case STRICT: - return getUnit(); - case DEFAULT: - case NULL: - if (_unitField!= null) { - return _unitField; - } else { - Object __rawValue = super._map.get("unit"); - _unitField = DataTemplateUtil.coerceEnumOutput(__rawValue, TimeUnit.class, TimeUnit.$UNKNOWN); - return _unitField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for unit - * - * @return - * Required field. Could be null for partial record. - * @see TimeOffset.Fields#unit - */ - @Nonnull - public TimeUnit getUnit() { - if (_unitField!= null) { - return _unitField; - } else { - Object __rawValue = super._map.get("unit"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("unit"); - } - _unitField = DataTemplateUtil.coerceEnumOutput(__rawValue, TimeUnit.class, TimeUnit.$UNKNOWN); - return _unitField; - } - } - - /** - * Setter for unit - * - * @see TimeOffset.Fields#unit - */ - public TimeOffset setUnit(TimeUnit value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setUnit(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field unit of com.linkedin.feathr.config.join.TimeOffset"); - } else { - CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); - _unitField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeUnit(); - } else { - CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); - _unitField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); - _unitField = value; - } - break; - } - return this; - } - - /** - * Setter for unit - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TimeOffset.Fields#unit - */ - public TimeOffset setUnit( - @Nonnull - TimeUnit value) { - if (value == null) { - throw new NullPointerException("Cannot set field unit of com.linkedin.feathr.config.join.TimeOffset to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); - _unitField = value; - } - return this; - } - - @Override - public TimeOffset clone() - throws CloneNotSupportedException - { - TimeOffset __clone = ((TimeOffset) super.clone()); - __clone.__changeListener = new TimeOffset.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public TimeOffset copy() - throws CloneNotSupportedException - { - TimeOffset __copy = ((TimeOffset) super.copy()); - __copy._unitField = null; - __copy._lengthField = null; - __copy.__changeListener = new TimeOffset.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final TimeOffset __objectRef; - - private ChangeListener(TimeOffset reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "unit": - __objectRef._unitField = null; - break; - case "length": - __objectRef._lengthField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Amount of the duration in TimeUnits. Can be positive or negative. - * - */ - public PathSpec length() { - return new PathSpec(getPathComponents(), "length"); - } - - /** - * Time unit for "length". For example, TimeUnit.DAY or TimeUnit.HOUR. - * - */ - public PathSpec unit() { - return new PathSpec(getPathComponents(), "unit"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(3); - } - - /** - * Amount of the duration in TimeUnits. Can be positive or negative. - * - */ - public TimeOffset.ProjectionMask withLength() { - getDataMap().put("length", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Time unit for "length". For example, TimeUnit.DAY or TimeUnit.HOUR. - * - */ - public TimeOffset.ProjectionMask withUnit() { - getDataMap().put("unit", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeUnit.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeUnit.java deleted file mode 100644 index b5ea832a6..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeUnit.java +++ /dev/null @@ -1,48 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import javax.annotation.Generated; -import com.linkedin.data.schema.EnumDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; - - -/** - * Unit of time used for defining a time range. - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimeUnit.pdl.") -public enum TimeUnit { - - - /** - * Daily format - * - */ - DAY, - - /** - * Hourly format - * - */ - HOUR, - - /** - * minute format, this can be used in simulate time delay - * - */ - MINUTE, - - /** - * second format, this can be used in simulate time delay - * - */ - SECOND, - $UNKNOWN; - private final static EnumDataSchema SCHEMA = ((EnumDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}", SchemaFormatType.PDL)); - - public static EnumDataSchema dataSchema() { - return SCHEMA; - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeWindow.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeWindow.java deleted file mode 100644 index 16e3aa965..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimeWindow.java +++ /dev/null @@ -1,412 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * Represents a length of time along with the corresponding time unit (DAY, HOUR). - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimeWindow.pdl.") -public class TimeWindow - extends RecordTemplate -{ - - private final static TimeWindow.Fields _fields = new TimeWindow.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Represents a length of time along with the corresponding time unit (DAY, HOUR).*/record TimeWindow{/**Amount of the duration in TimeUnits. Can be greater or equal to 1.*/@validate.positive,length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}", SchemaFormatType.PDL)); - private Long _lengthField = null; - private TimeUnit _unitField = null; - private TimeWindow.ChangeListener __changeListener = new TimeWindow.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Length = SCHEMA.getField("length"); - private final static RecordDataSchema.Field FIELD_Unit = SCHEMA.getField("unit"); - - public TimeWindow() { - super(new DataMap(3, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public TimeWindow(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static TimeWindow.Fields fields() { - return _fields; - } - - public static TimeWindow.ProjectionMask createMask() { - return new TimeWindow.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for length - * - * @see TimeWindow.Fields#length - */ - public boolean hasLength() { - if (_lengthField!= null) { - return true; - } - return super._map.containsKey("length"); - } - - /** - * Remover for length - * - * @see TimeWindow.Fields#length - */ - public void removeLength() { - super._map.remove("length"); - } - - /** - * Getter for length - * - * @see TimeWindow.Fields#length - */ - public Long getLength(GetMode mode) { - switch (mode) { - case STRICT: - return getLength(); - case DEFAULT: - case NULL: - if (_lengthField!= null) { - return _lengthField; - } else { - Object __rawValue = super._map.get("length"); - _lengthField = DataTemplateUtil.coerceLongOutput(__rawValue); - return _lengthField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for length - * - * @return - * Required field. Could be null for partial record. - * @see TimeWindow.Fields#length - */ - @Nonnull - public Long getLength() { - if (_lengthField!= null) { - return _lengthField; - } else { - Object __rawValue = super._map.get("length"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("length"); - } - _lengthField = DataTemplateUtil.coerceLongOutput(__rawValue); - return _lengthField; - } - } - - /** - * Setter for length - * - * @see TimeWindow.Fields#length - */ - public TimeWindow setLength(Long value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setLength(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field length of com.linkedin.feathr.config.join.TimeWindow"); - } else { - CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); - _lengthField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeLength(); - } else { - CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); - _lengthField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); - _lengthField = value; - } - break; - } - return this; - } - - /** - * Setter for length - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TimeWindow.Fields#length - */ - public TimeWindow setLength( - @Nonnull - Long value) { - if (value == null) { - throw new NullPointerException("Cannot set field length of com.linkedin.feathr.config.join.TimeWindow to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); - _lengthField = value; - } - return this; - } - - /** - * Setter for length - * - * @see TimeWindow.Fields#length - */ - public TimeWindow setLength(long value) { - CheckedUtil.putWithoutChecking(super._map, "length", DataTemplateUtil.coerceLongInput(value)); - _lengthField = value; - return this; - } - - /** - * Existence checker for unit - * - * @see TimeWindow.Fields#unit - */ - public boolean hasUnit() { - if (_unitField!= null) { - return true; - } - return super._map.containsKey("unit"); - } - - /** - * Remover for unit - * - * @see TimeWindow.Fields#unit - */ - public void removeUnit() { - super._map.remove("unit"); - } - - /** - * Getter for unit - * - * @see TimeWindow.Fields#unit - */ - public TimeUnit getUnit(GetMode mode) { - switch (mode) { - case STRICT: - return getUnit(); - case DEFAULT: - case NULL: - if (_unitField!= null) { - return _unitField; - } else { - Object __rawValue = super._map.get("unit"); - _unitField = DataTemplateUtil.coerceEnumOutput(__rawValue, TimeUnit.class, TimeUnit.$UNKNOWN); - return _unitField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for unit - * - * @return - * Required field. Could be null for partial record. - * @see TimeWindow.Fields#unit - */ - @Nonnull - public TimeUnit getUnit() { - if (_unitField!= null) { - return _unitField; - } else { - Object __rawValue = super._map.get("unit"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("unit"); - } - _unitField = DataTemplateUtil.coerceEnumOutput(__rawValue, TimeUnit.class, TimeUnit.$UNKNOWN); - return _unitField; - } - } - - /** - * Setter for unit - * - * @see TimeWindow.Fields#unit - */ - public TimeWindow setUnit(TimeUnit value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setUnit(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field unit of com.linkedin.feathr.config.join.TimeWindow"); - } else { - CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); - _unitField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeUnit(); - } else { - CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); - _unitField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); - _unitField = value; - } - break; - } - return this; - } - - /** - * Setter for unit - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TimeWindow.Fields#unit - */ - public TimeWindow setUnit( - @Nonnull - TimeUnit value) { - if (value == null) { - throw new NullPointerException("Cannot set field unit of com.linkedin.feathr.config.join.TimeWindow to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "unit", value.name()); - _unitField = value; - } - return this; - } - - @Override - public TimeWindow clone() - throws CloneNotSupportedException - { - TimeWindow __clone = ((TimeWindow) super.clone()); - __clone.__changeListener = new TimeWindow.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public TimeWindow copy() - throws CloneNotSupportedException - { - TimeWindow __copy = ((TimeWindow) super.copy()); - __copy._unitField = null; - __copy._lengthField = null; - __copy.__changeListener = new TimeWindow.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final TimeWindow __objectRef; - - private ChangeListener(TimeWindow reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "unit": - __objectRef._unitField = null; - break; - case "length": - __objectRef._lengthField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Amount of the duration in TimeUnits. Can be greater or equal to 1. - * - */ - public PathSpec length() { - return new PathSpec(getPathComponents(), "length"); - } - - /** - * Time unit for "length". For example, TimeUnit.DAY or TimeUnit.HOUR. - * - */ - public PathSpec unit() { - return new PathSpec(getPathComponents(), "unit"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(3); - } - - /** - * Amount of the duration in TimeUnits. Can be greater or equal to 1. - * - */ - public TimeWindow.ProjectionMask withLength() { - getDataMap().put("length", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Time unit for "length". For example, TimeUnit.DAY or TimeUnit.HOUR. - * - */ - public TimeWindow.ProjectionMask withUnit() { - getDataMap().put("unit", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColJoinTimeSettings.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColJoinTimeSettings.java deleted file mode 100644 index 3e87c616d..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColJoinTimeSettings.java +++ /dev/null @@ -1,432 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; - - -/** - * Settings needed when the input data has a timestamp which should be used for the join. - * joinTimeSettings: { - * timestampColumn: { - * def: timestamp - * format: yyyy/MM/dd - * } - * simulateTimeDelay: 1d - * } - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimestampColJoinTimeSettings.pdl.") -public class TimestampColJoinTimeSettings - extends RecordTemplate -{ - - private final static TimestampColJoinTimeSettings.Fields _fields = new TimestampColJoinTimeSettings.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Settings needed when the input data has a timestamp which should be used for the join.\r\njoinTimeSettings: {\r\n timestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }\r\n simulateTimeDelay: 1d\r\n}*/record TimestampColJoinTimeSettings{/**The timestamp column name and timeformat which should be used for joining with the feature data.\r\nRefer to [[TimestampColumn]].\r\nExample, TimestampColumn: {\r\n def: timestamp\r\n format: yyyy/MM/dd\r\n }*/timestampColumn:/**Timestamp column of the input featureiized dataset, which is to be used for the join.\r\ntimestampColumn: {\r\n def: timestamp\r\n format: yyyyMMdd\r\n }*/record TimestampColumn{/**The definiton of the timestamp column, which can be a sql expression involving the timestamp column\r\nor just the column name\r\nExample:- definition: timestamp, timestamp + 10000000.*/definition:union[columnName:string,sparkSqlExpression:/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}]/**Format of the timestamp column. Must confer to java's timestampFormatter or can be\r\nepoch or epoch_millis.\r\nExample:- epoch, epoch_millis, yyyy/MM/dd*/format:/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string}/**An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted\r\nfrom the input data timestamp while joining with the feature data.\r\nWe do support negative time delays.*/simulateTimeDelay:optional/**TimeOffset is the amount of time we need to push back the current time wrt a reference time. Since, reference time can\r\nbe any time in the past also, we do allow a positive or negative offset length.\r\n offset - 1 day implies the previous from the reference day.*/record TimeOffset{/**Amount of the duration in TimeUnits. Can be positive or negative.*/length:long/**Time unit for \"length\". For example, TimeUnit.DAY or TimeUnit.HOUR.*/unit:/**Unit of time used for defining a time range.*/enum TimeUnit{/**Daily format*/DAY/**Hourly format*/HOUR/**minute format, this can be used in simulate time delay*/MINUTE/**second format, this can be used in simulate time delay*/SECOND}}}", SchemaFormatType.PDL)); - private TimestampColumn _timestampColumnField = null; - private TimeOffset _simulateTimeDelayField = null; - private TimestampColJoinTimeSettings.ChangeListener __changeListener = new TimestampColJoinTimeSettings.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_TimestampColumn = SCHEMA.getField("timestampColumn"); - private final static RecordDataSchema.Field FIELD_SimulateTimeDelay = SCHEMA.getField("simulateTimeDelay"); - - public TimestampColJoinTimeSettings() { - super(new DataMap(3, 0.75F), SCHEMA, 3); - addChangeListener(__changeListener); - } - - public TimestampColJoinTimeSettings(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static TimestampColJoinTimeSettings.Fields fields() { - return _fields; - } - - public static TimestampColJoinTimeSettings.ProjectionMask createMask() { - return new TimestampColJoinTimeSettings.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for timestampColumn - * - * @see TimestampColJoinTimeSettings.Fields#timestampColumn - */ - public boolean hasTimestampColumn() { - if (_timestampColumnField!= null) { - return true; - } - return super._map.containsKey("timestampColumn"); - } - - /** - * Remover for timestampColumn - * - * @see TimestampColJoinTimeSettings.Fields#timestampColumn - */ - public void removeTimestampColumn() { - super._map.remove("timestampColumn"); - } - - /** - * Getter for timestampColumn - * - * @see TimestampColJoinTimeSettings.Fields#timestampColumn - */ - public TimestampColumn getTimestampColumn(GetMode mode) { - switch (mode) { - case STRICT: - return getTimestampColumn(); - case DEFAULT: - case NULL: - if (_timestampColumnField!= null) { - return _timestampColumnField; - } else { - Object __rawValue = super._map.get("timestampColumn"); - _timestampColumnField = ((__rawValue == null)?null:new TimestampColumn(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _timestampColumnField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for timestampColumn - * - * @return - * Required field. Could be null for partial record. - * @see TimestampColJoinTimeSettings.Fields#timestampColumn - */ - @Nonnull - public TimestampColumn getTimestampColumn() { - if (_timestampColumnField!= null) { - return _timestampColumnField; - } else { - Object __rawValue = super._map.get("timestampColumn"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("timestampColumn"); - } - _timestampColumnField = ((__rawValue == null)?null:new TimestampColumn(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _timestampColumnField; - } - } - - /** - * Setter for timestampColumn - * - * @see TimestampColJoinTimeSettings.Fields#timestampColumn - */ - public TimestampColJoinTimeSettings setTimestampColumn(TimestampColumn value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setTimestampColumn(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field timestampColumn of com.linkedin.feathr.config.join.TimestampColJoinTimeSettings"); - } else { - CheckedUtil.putWithoutChecking(super._map, "timestampColumn", value.data()); - _timestampColumnField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeTimestampColumn(); - } else { - CheckedUtil.putWithoutChecking(super._map, "timestampColumn", value.data()); - _timestampColumnField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "timestampColumn", value.data()); - _timestampColumnField = value; - } - break; - } - return this; - } - - /** - * Setter for timestampColumn - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TimestampColJoinTimeSettings.Fields#timestampColumn - */ - public TimestampColJoinTimeSettings setTimestampColumn( - @Nonnull - TimestampColumn value) { - if (value == null) { - throw new NullPointerException("Cannot set field timestampColumn of com.linkedin.feathr.config.join.TimestampColJoinTimeSettings to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "timestampColumn", value.data()); - _timestampColumnField = value; - } - return this; - } - - /** - * Existence checker for simulateTimeDelay - * - * @see TimestampColJoinTimeSettings.Fields#simulateTimeDelay - */ - public boolean hasSimulateTimeDelay() { - if (_simulateTimeDelayField!= null) { - return true; - } - return super._map.containsKey("simulateTimeDelay"); - } - - /** - * Remover for simulateTimeDelay - * - * @see TimestampColJoinTimeSettings.Fields#simulateTimeDelay - */ - public void removeSimulateTimeDelay() { - super._map.remove("simulateTimeDelay"); - } - - /** - * Getter for simulateTimeDelay - * - * @see TimestampColJoinTimeSettings.Fields#simulateTimeDelay - */ - public TimeOffset getSimulateTimeDelay(GetMode mode) { - return getSimulateTimeDelay(); - } - - /** - * Getter for simulateTimeDelay - * - * @return - * Optional field. Always check for null. - * @see TimestampColJoinTimeSettings.Fields#simulateTimeDelay - */ - @Nullable - public TimeOffset getSimulateTimeDelay() { - if (_simulateTimeDelayField!= null) { - return _simulateTimeDelayField; - } else { - Object __rawValue = super._map.get("simulateTimeDelay"); - _simulateTimeDelayField = ((__rawValue == null)?null:new TimeOffset(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _simulateTimeDelayField; - } - } - - /** - * Setter for simulateTimeDelay - * - * @see TimestampColJoinTimeSettings.Fields#simulateTimeDelay - */ - public TimestampColJoinTimeSettings setSimulateTimeDelay(TimeOffset value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setSimulateTimeDelay(value); - case REMOVE_OPTIONAL_IF_NULL: - case REMOVE_IF_NULL: - if (value == null) { - removeSimulateTimeDelay(); - } else { - CheckedUtil.putWithoutChecking(super._map, "simulateTimeDelay", value.data()); - _simulateTimeDelayField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "simulateTimeDelay", value.data()); - _simulateTimeDelayField = value; - } - break; - } - return this; - } - - /** - * Setter for simulateTimeDelay - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TimestampColJoinTimeSettings.Fields#simulateTimeDelay - */ - public TimestampColJoinTimeSettings setSimulateTimeDelay( - @Nonnull - TimeOffset value) { - if (value == null) { - throw new NullPointerException("Cannot set field simulateTimeDelay of com.linkedin.feathr.config.join.TimestampColJoinTimeSettings to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "simulateTimeDelay", value.data()); - _simulateTimeDelayField = value; - } - return this; - } - - @Override - public TimestampColJoinTimeSettings clone() - throws CloneNotSupportedException - { - TimestampColJoinTimeSettings __clone = ((TimestampColJoinTimeSettings) super.clone()); - __clone.__changeListener = new TimestampColJoinTimeSettings.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public TimestampColJoinTimeSettings copy() - throws CloneNotSupportedException - { - TimestampColJoinTimeSettings __copy = ((TimestampColJoinTimeSettings) super.copy()); - __copy._simulateTimeDelayField = null; - __copy._timestampColumnField = null; - __copy.__changeListener = new TimestampColJoinTimeSettings.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final TimestampColJoinTimeSettings __objectRef; - - private ChangeListener(TimestampColJoinTimeSettings reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "simulateTimeDelay": - __objectRef._simulateTimeDelayField = null; - break; - case "timestampColumn": - __objectRef._timestampColumnField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The timestamp column name and timeformat which should be used for joining with the feature data. - * Refer to [[TimestampColumn]]. - * Example, TimestampColumn: { - * def: timestamp - * format: yyyy/MM/dd - * } - * - */ - public com.linkedin.feathr.config.join.TimestampColumn.Fields timestampColumn() { - return new com.linkedin.feathr.config.join.TimestampColumn.Fields(getPathComponents(), "timestampColumn"); - } - - /** - * An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted - * from the input data timestamp while joining with the feature data. - * We do support negative time delays. - * - */ - public com.linkedin.feathr.config.join.TimeOffset.Fields simulateTimeDelay() { - return new com.linkedin.feathr.config.join.TimeOffset.Fields(getPathComponents(), "simulateTimeDelay"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.TimestampColumn.ProjectionMask _timestampColumnMask; - private com.linkedin.feathr.config.join.TimeOffset.ProjectionMask _simulateTimeDelayMask; - - ProjectionMask() { - super(3); - } - - /** - * The timestamp column name and timeformat which should be used for joining with the feature data. - * Refer to [[TimestampColumn]]. - * Example, TimestampColumn: { - * def: timestamp - * format: yyyy/MM/dd - * } - * - */ - public TimestampColJoinTimeSettings.ProjectionMask withTimestampColumn(Function nestedMask) { - _timestampColumnMask = nestedMask.apply(((_timestampColumnMask == null)?TimestampColumn.createMask():_timestampColumnMask)); - getDataMap().put("timestampColumn", _timestampColumnMask.getDataMap()); - return this; - } - - /** - * The timestamp column name and timeformat which should be used for joining with the feature data. - * Refer to [[TimestampColumn]]. - * Example, TimestampColumn: { - * def: timestamp - * format: yyyy/MM/dd - * } - * - */ - public TimestampColJoinTimeSettings.ProjectionMask withTimestampColumn() { - _timestampColumnMask = null; - getDataMap().put("timestampColumn", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted - * from the input data timestamp while joining with the feature data. - * We do support negative time delays. - * - */ - public TimestampColJoinTimeSettings.ProjectionMask withSimulateTimeDelay(Function nestedMask) { - _simulateTimeDelayMask = nestedMask.apply(((_simulateTimeDelayMask == null)?TimeOffset.createMask():_simulateTimeDelayMask)); - getDataMap().put("simulateTimeDelay", _simulateTimeDelayMask.getDataMap()); - return this; - } - - /** - * An optional simulate time delay parameter which can be set by the user. Indicates the amount of time that is to subtracted - * from the input data timestamp while joining with the feature data. - * We do support negative time delays. - * - */ - public TimestampColJoinTimeSettings.ProjectionMask withSimulateTimeDelay() { - _simulateTimeDelayMask = null; - getDataMap().put("simulateTimeDelay", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColumn.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColumn.java deleted file mode 100644 index dae1adacc..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/TimestampColumn.java +++ /dev/null @@ -1,609 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import java.util.function.Function; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.DataSchema; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.schema.UnionDataSchema; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.RequiredFieldNotPresentException; -import com.linkedin.data.template.SetMode; -import com.linkedin.data.template.UnionTemplate; - - -/** - * Timestamp column of the input featureiized dataset, which is to be used for the join. - * timestampColumn: { - * def: timestamp - * format: yyyyMMdd - * } - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimestampColumn.pdl.") -public class TimestampColumn - extends RecordTemplate -{ - - private final static TimestampColumn.Fields _fields = new TimestampColumn.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Timestamp column of the input featureiized dataset, which is to be used for the join.\r\ntimestampColumn: {\r\n def: timestamp\r\n format: yyyyMMdd\r\n }*/record TimestampColumn{/**The definiton of the timestamp column, which can be a sql expression involving the timestamp column\r\nor just the column name\r\nExample:- definition: timestamp, timestamp + 10000000.*/definition:union[columnName:string,sparkSqlExpression:/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}]/**Format of the timestamp column. Must confer to java's timestampFormatter or can be\r\nepoch or epoch_millis.\r\nExample:- epoch, epoch_millis, yyyy/MM/dd*/format:/**The timeformat, which accepts the formats parsed by the DateTimeFormatter java class or epoch or epoch_millis. However in future, we can have\r\nthe option of a stronger type. Example, dd/MM/yyyy, yyyy-MM-dd, epoch, epoch_millis, etc.*/typeref TimeFormat=string}", SchemaFormatType.PDL)); - private TimestampColumn.Definition _definitionField = null; - private String _formatField = null; - private TimestampColumn.ChangeListener __changeListener = new TimestampColumn.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_Definition = SCHEMA.getField("definition"); - private final static RecordDataSchema.Field FIELD_Format = SCHEMA.getField("format"); - - public TimestampColumn() { - super(new DataMap(3, 0.75F), SCHEMA, 2); - addChangeListener(__changeListener); - } - - public TimestampColumn(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static TimestampColumn.Fields fields() { - return _fields; - } - - public static TimestampColumn.ProjectionMask createMask() { - return new TimestampColumn.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for definition - * - * @see TimestampColumn.Fields#definition - */ - public boolean hasDefinition() { - if (_definitionField!= null) { - return true; - } - return super._map.containsKey("definition"); - } - - /** - * Remover for definition - * - * @see TimestampColumn.Fields#definition - */ - public void removeDefinition() { - super._map.remove("definition"); - } - - /** - * Getter for definition - * - * @see TimestampColumn.Fields#definition - */ - public TimestampColumn.Definition getDefinition(GetMode mode) { - switch (mode) { - case STRICT: - return getDefinition(); - case DEFAULT: - case NULL: - if (_definitionField!= null) { - return _definitionField; - } else { - Object __rawValue = super._map.get("definition"); - _definitionField = ((__rawValue == null)?null:new TimestampColumn.Definition(__rawValue)); - return _definitionField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for definition - * - * @return - * Required field. Could be null for partial record. - * @see TimestampColumn.Fields#definition - */ - @Nonnull - public TimestampColumn.Definition getDefinition() { - if (_definitionField!= null) { - return _definitionField; - } else { - Object __rawValue = super._map.get("definition"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("definition"); - } - _definitionField = ((__rawValue == null)?null:new TimestampColumn.Definition(__rawValue)); - return _definitionField; - } - } - - /** - * Setter for definition - * - * @see TimestampColumn.Fields#definition - */ - public TimestampColumn setDefinition(TimestampColumn.Definition value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setDefinition(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field definition of com.linkedin.feathr.config.join.TimestampColumn"); - } else { - CheckedUtil.putWithoutChecking(super._map, "definition", value.data()); - _definitionField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeDefinition(); - } else { - CheckedUtil.putWithoutChecking(super._map, "definition", value.data()); - _definitionField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "definition", value.data()); - _definitionField = value; - } - break; - } - return this; - } - - /** - * Setter for definition - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TimestampColumn.Fields#definition - */ - public TimestampColumn setDefinition( - @Nonnull - TimestampColumn.Definition value) { - if (value == null) { - throw new NullPointerException("Cannot set field definition of com.linkedin.feathr.config.join.TimestampColumn to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "definition", value.data()); - _definitionField = value; - } - return this; - } - - /** - * Existence checker for format - * - * @see TimestampColumn.Fields#format - */ - public boolean hasFormat() { - if (_formatField!= null) { - return true; - } - return super._map.containsKey("format"); - } - - /** - * Remover for format - * - * @see TimestampColumn.Fields#format - */ - public void removeFormat() { - super._map.remove("format"); - } - - /** - * Getter for format - * - * @see TimestampColumn.Fields#format - */ - public String getFormat(GetMode mode) { - switch (mode) { - case STRICT: - return getFormat(); - case DEFAULT: - case NULL: - if (_formatField!= null) { - return _formatField; - } else { - Object __rawValue = super._map.get("format"); - _formatField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _formatField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for format - * - * @return - * Required field. Could be null for partial record. - * @see TimestampColumn.Fields#format - */ - @Nonnull - public String getFormat() { - if (_formatField!= null) { - return _formatField; - } else { - Object __rawValue = super._map.get("format"); - if (__rawValue == null) { - throw new RequiredFieldNotPresentException("format"); - } - _formatField = DataTemplateUtil.coerceStringOutput(__rawValue); - return _formatField; - } - } - - /** - * Setter for format - * - * @see TimestampColumn.Fields#format - */ - public TimestampColumn setFormat(String value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setFormat(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field format of com.linkedin.feathr.config.join.TimestampColumn"); - } else { - CheckedUtil.putWithoutChecking(super._map, "format", value); - _formatField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeFormat(); - } else { - CheckedUtil.putWithoutChecking(super._map, "format", value); - _formatField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "format", value); - _formatField = value; - } - break; - } - return this; - } - - /** - * Setter for format - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see TimestampColumn.Fields#format - */ - public TimestampColumn setFormat( - @Nonnull - String value) { - if (value == null) { - throw new NullPointerException("Cannot set field format of com.linkedin.feathr.config.join.TimestampColumn to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "format", value); - _formatField = value; - } - return this; - } - - @Override - public TimestampColumn clone() - throws CloneNotSupportedException - { - TimestampColumn __clone = ((TimestampColumn) super.clone()); - __clone.__changeListener = new TimestampColumn.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public TimestampColumn copy() - throws CloneNotSupportedException - { - TimestampColumn __copy = ((TimestampColumn) super.copy()); - __copy._formatField = null; - __copy._definitionField = null; - __copy.__changeListener = new TimestampColumn.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final TimestampColumn __objectRef; - - private ChangeListener(TimestampColumn reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "format": - __objectRef._formatField = null; - break; - case "definition": - __objectRef._definitionField = null; - break; - } - } - - } - - @Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\TimestampColumn.pdl.") - public static class Definition - extends UnionTemplate - { - - private final static UnionDataSchema SCHEMA = ((UnionDataSchema) DataTemplateUtil.parseSchema("union[columnName:string,sparkSqlExpression:{namespace com.linkedin.feathr.config.join/**An expression in Spark SQL.*/record SparkSqlExpression{/**The Spark SQL expression.*/expression:string}}]", SchemaFormatType.PDL)); - private String _columnNameMember = null; - private com.linkedin.feathr.config.join.SparkSqlExpression _sparkSqlExpressionMember = null; - private TimestampColumn.Definition.ChangeListener __changeListener = new TimestampColumn.Definition.ChangeListener(this); - private final static DataSchema MEMBER_ColumnName = SCHEMA.getTypeByMemberKey("columnName"); - private final static DataSchema MEMBER_SparkSqlExpression = SCHEMA.getTypeByMemberKey("sparkSqlExpression"); - - public Definition() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public Definition(Object data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UnionDataSchema dataSchema() { - return SCHEMA; - } - - public static TimestampColumn.Definition createWithColumnName(String value) { - TimestampColumn.Definition newUnion = new TimestampColumn.Definition(); - newUnion.setColumnName(value); - return newUnion; - } - - public boolean isColumnName() { - return memberIs("columnName"); - } - - public String getColumnName() { - checkNotNull(); - if (_columnNameMember!= null) { - return _columnNameMember; - } - Object __rawValue = super._map.get("columnName"); - _columnNameMember = DataTemplateUtil.coerceStringOutput(__rawValue); - return _columnNameMember; - } - - public void setColumnName(String value) { - checkNotNull(); - super._map.clear(); - _columnNameMember = value; - CheckedUtil.putWithoutChecking(super._map, "columnName", value); - } - - public static TimestampColumn.Definition createWithSparkSqlExpression(com.linkedin.feathr.config.join.SparkSqlExpression value) { - TimestampColumn.Definition newUnion = new TimestampColumn.Definition(); - newUnion.setSparkSqlExpression(value); - return newUnion; - } - - public boolean isSparkSqlExpression() { - return memberIs("sparkSqlExpression"); - } - - public com.linkedin.feathr.config.join.SparkSqlExpression getSparkSqlExpression() { - checkNotNull(); - if (_sparkSqlExpressionMember!= null) { - return _sparkSqlExpressionMember; - } - Object __rawValue = super._map.get("sparkSqlExpression"); - _sparkSqlExpressionMember = ((__rawValue == null)?null:new com.linkedin.feathr.config.join.SparkSqlExpression(DataTemplateUtil.castOrThrow(__rawValue, DataMap.class))); - return _sparkSqlExpressionMember; - } - - public void setSparkSqlExpression(com.linkedin.feathr.config.join.SparkSqlExpression value) { - checkNotNull(); - super._map.clear(); - _sparkSqlExpressionMember = value; - CheckedUtil.putWithoutChecking(super._map, "sparkSqlExpression", value.data()); - } - - public static TimestampColumn.Definition.ProjectionMask createMask() { - return new TimestampColumn.Definition.ProjectionMask(); - } - - @Override - public TimestampColumn.Definition clone() - throws CloneNotSupportedException - { - TimestampColumn.Definition __clone = ((TimestampColumn.Definition) super.clone()); - __clone.__changeListener = new TimestampColumn.Definition.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public TimestampColumn.Definition copy() - throws CloneNotSupportedException - { - TimestampColumn.Definition __copy = ((TimestampColumn.Definition) super.copy()); - __copy._sparkSqlExpressionMember = null; - __copy._columnNameMember = null; - __copy.__changeListener = new TimestampColumn.Definition.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final TimestampColumn.Definition __objectRef; - - private ChangeListener(TimestampColumn.Definition reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "sparkSqlExpression": - __objectRef._sparkSqlExpressionMember = null; - break; - case "columnName": - __objectRef._columnNameMember = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - public PathSpec ColumnName() { - return new PathSpec(getPathComponents(), "columnName"); - } - - public com.linkedin.feathr.config.join.SparkSqlExpression.Fields SparkSqlExpression() { - return new com.linkedin.feathr.config.join.SparkSqlExpression.Fields(getPathComponents(), "sparkSqlExpression"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.SparkSqlExpression.ProjectionMask _SparkSqlExpressionMask; - - ProjectionMask() { - super(3); - } - - public TimestampColumn.Definition.ProjectionMask withColumnName() { - getDataMap().put("columnName", MaskMap.POSITIVE_MASK); - return this; - } - - public TimestampColumn.Definition.ProjectionMask withSparkSqlExpression(Function nestedMask) { - _SparkSqlExpressionMask = nestedMask.apply(((_SparkSqlExpressionMask == null)?com.linkedin.feathr.config.join.SparkSqlExpression.createMask():_SparkSqlExpressionMask)); - getDataMap().put("sparkSqlExpression", _SparkSqlExpressionMask.getDataMap()); - return this; - } - - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * The definiton of the timestamp column, which can be a sql expression involving the timestamp column - * or just the column name - * Example:- definition: timestamp, timestamp + 10000000. - * - */ - public com.linkedin.feathr.config.join.TimestampColumn.Definition.Fields definition() { - return new com.linkedin.feathr.config.join.TimestampColumn.Definition.Fields(getPathComponents(), "definition"); - } - - /** - * Format of the timestamp column. Must confer to java's timestampFormatter or can be - * epoch or epoch_millis. - * Example:- epoch, epoch_millis, yyyy/MM/dd - * - */ - public PathSpec format() { - return new PathSpec(getPathComponents(), "format"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - private com.linkedin.feathr.config.join.TimestampColumn.Definition.ProjectionMask _definitionMask; - - ProjectionMask() { - super(3); - } - - /** - * The definiton of the timestamp column, which can be a sql expression involving the timestamp column - * or just the column name - * Example:- definition: timestamp, timestamp + 10000000. - * - */ - public TimestampColumn.ProjectionMask withDefinition(Function nestedMask) { - _definitionMask = nestedMask.apply(((_definitionMask == null)?TimestampColumn.Definition.createMask():_definitionMask)); - getDataMap().put("definition", _definitionMask.getDataMap()); - return this; - } - - /** - * The definiton of the timestamp column, which can be a sql expression involving the timestamp column - * or just the column name - * Example:- definition: timestamp, timestamp + 10000000. - * - */ - public TimestampColumn.ProjectionMask withDefinition() { - _definitionMask = null; - getDataMap().put("definition", MaskMap.POSITIVE_MASK); - return this; - } - - /** - * Format of the timestamp column. Must confer to java's timestampFormatter or can be - * epoch or epoch_millis. - * Example:- epoch, epoch_millis, yyyy/MM/dd - * - */ - public TimestampColumn.ProjectionMask withFormat() { - getDataMap().put("format", MaskMap.POSITIVE_MASK); - return this; - } - - } - -} diff --git a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/UseLatestJoinTimeSettings.java b/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/UseLatestJoinTimeSettings.java deleted file mode 100644 index 7e796b319..000000000 --- a/feathr-data-models/src/mainGeneratedDataTemplate/java/com/linkedin/feathr/config/join/UseLatestJoinTimeSettings.java +++ /dev/null @@ -1,280 +0,0 @@ - -package com.linkedin.feathr.config.join; - -import java.util.List; -import javax.annotation.Generated; -import javax.annotation.Nonnull; -import com.linkedin.data.DataMap; -import com.linkedin.data.collections.CheckedUtil; -import com.linkedin.data.schema.MaskMap; -import com.linkedin.data.schema.PathSpec; -import com.linkedin.data.schema.RecordDataSchema; -import com.linkedin.data.schema.SchemaFormatType; -import com.linkedin.data.template.DataTemplateUtil; -import com.linkedin.data.template.GetMode; -import com.linkedin.data.template.RecordTemplate; -import com.linkedin.data.template.SetMode; - - -/** - * Settings needed when the input data is to be joined with the latest available feature data. - * joinTimeSettings: { - * useLatestFeatureData: true - * } - * - */ -@Generated(value = "com.linkedin.pegasus.generator.JavaCodeUtil", comments = "Rest.li Data Template. Generated from feathr-data-models\\src\\main\\pegasus\\com\\linkedin\\feathr\\config\\join\\UseLatestJoinTimeSettings.pdl.") -public class UseLatestJoinTimeSettings - extends RecordTemplate -{ - - private final static UseLatestJoinTimeSettings.Fields _fields = new UseLatestJoinTimeSettings.Fields(); - private final static RecordDataSchema SCHEMA = ((RecordDataSchema) DataTemplateUtil.parseSchema("namespace com.linkedin.feathr.config.join/**Settings needed when the input data is to be joined with the latest available feature data.\r\njoinTimeSettings: {\r\n useLatestFeatureData: true\r\n}*/record UseLatestJoinTimeSettings{/**Boolean value, if set to true, indicates that the latest available feature data is to be used for joining.\r\nWhen useLatestFeatureData is set, there should be no other time-based parameters.*/useLatestFeatureData:boolean=true}", SchemaFormatType.PDL)); - private Boolean _useLatestFeatureDataField = null; - private UseLatestJoinTimeSettings.ChangeListener __changeListener = new UseLatestJoinTimeSettings.ChangeListener(this); - private final static RecordDataSchema.Field FIELD_UseLatestFeatureData = SCHEMA.getField("useLatestFeatureData"); - private final static Boolean DEFAULT_UseLatestFeatureData; - - static { - DEFAULT_UseLatestFeatureData = DataTemplateUtil.coerceBooleanOutput(FIELD_UseLatestFeatureData.getDefault()); - } - - public UseLatestJoinTimeSettings() { - super(new DataMap(2, 0.75F), SCHEMA); - addChangeListener(__changeListener); - } - - public UseLatestJoinTimeSettings(DataMap data) { - super(data, SCHEMA); - addChangeListener(__changeListener); - } - - public static UseLatestJoinTimeSettings.Fields fields() { - return _fields; - } - - public static UseLatestJoinTimeSettings.ProjectionMask createMask() { - return new UseLatestJoinTimeSettings.ProjectionMask(); - } - - public static RecordDataSchema dataSchema() { - return SCHEMA; - } - - /** - * Existence checker for useLatestFeatureData - * - * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData - */ - public boolean hasUseLatestFeatureData() { - if (_useLatestFeatureDataField!= null) { - return true; - } - return super._map.containsKey("useLatestFeatureData"); - } - - /** - * Remover for useLatestFeatureData - * - * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData - */ - public void removeUseLatestFeatureData() { - super._map.remove("useLatestFeatureData"); - } - - /** - * Getter for useLatestFeatureData - * - * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData - */ - public Boolean isUseLatestFeatureData(GetMode mode) { - switch (mode) { - case STRICT: - case DEFAULT: - return isUseLatestFeatureData(); - case NULL: - if (_useLatestFeatureDataField!= null) { - return _useLatestFeatureDataField; - } else { - Object __rawValue = super._map.get("useLatestFeatureData"); - _useLatestFeatureDataField = DataTemplateUtil.coerceBooleanOutput(__rawValue); - return _useLatestFeatureDataField; - } - } - throw new IllegalStateException(("Unknown mode "+ mode)); - } - - /** - * Getter for useLatestFeatureData - * - * @return - * Required field. Could be null for partial record. - * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData - */ - @Nonnull - public Boolean isUseLatestFeatureData() { - if (_useLatestFeatureDataField!= null) { - return _useLatestFeatureDataField; - } else { - Object __rawValue = super._map.get("useLatestFeatureData"); - if (__rawValue == null) { - return DEFAULT_UseLatestFeatureData; - } - _useLatestFeatureDataField = DataTemplateUtil.coerceBooleanOutput(__rawValue); - return _useLatestFeatureDataField; - } - } - - /** - * Setter for useLatestFeatureData - * - * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData - */ - public UseLatestJoinTimeSettings setUseLatestFeatureData(Boolean value, SetMode mode) { - switch (mode) { - case DISALLOW_NULL: - return setUseLatestFeatureData(value); - case REMOVE_OPTIONAL_IF_NULL: - if (value == null) { - throw new IllegalArgumentException("Cannot remove mandatory field useLatestFeatureData of com.linkedin.feathr.config.join.UseLatestJoinTimeSettings"); - } else { - CheckedUtil.putWithoutChecking(super._map, "useLatestFeatureData", value); - _useLatestFeatureDataField = value; - } - break; - case REMOVE_IF_NULL: - if (value == null) { - removeUseLatestFeatureData(); - } else { - CheckedUtil.putWithoutChecking(super._map, "useLatestFeatureData", value); - _useLatestFeatureDataField = value; - } - break; - case IGNORE_NULL: - if (value!= null) { - CheckedUtil.putWithoutChecking(super._map, "useLatestFeatureData", value); - _useLatestFeatureDataField = value; - } - break; - } - return this; - } - - /** - * Setter for useLatestFeatureData - * - * @param value - * Must not be null. For more control, use setters with mode instead. - * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData - */ - public UseLatestJoinTimeSettings setUseLatestFeatureData( - @Nonnull - Boolean value) { - if (value == null) { - throw new NullPointerException("Cannot set field useLatestFeatureData of com.linkedin.feathr.config.join.UseLatestJoinTimeSettings to null"); - } else { - CheckedUtil.putWithoutChecking(super._map, "useLatestFeatureData", value); - _useLatestFeatureDataField = value; - } - return this; - } - - /** - * Setter for useLatestFeatureData - * - * @see UseLatestJoinTimeSettings.Fields#useLatestFeatureData - */ - public UseLatestJoinTimeSettings setUseLatestFeatureData(boolean value) { - CheckedUtil.putWithoutChecking(super._map, "useLatestFeatureData", value); - _useLatestFeatureDataField = value; - return this; - } - - @Override - public UseLatestJoinTimeSettings clone() - throws CloneNotSupportedException - { - UseLatestJoinTimeSettings __clone = ((UseLatestJoinTimeSettings) super.clone()); - __clone.__changeListener = new UseLatestJoinTimeSettings.ChangeListener(__clone); - __clone.addChangeListener(__clone.__changeListener); - return __clone; - } - - @Override - public UseLatestJoinTimeSettings copy() - throws CloneNotSupportedException - { - UseLatestJoinTimeSettings __copy = ((UseLatestJoinTimeSettings) super.copy()); - __copy._useLatestFeatureDataField = null; - __copy.__changeListener = new UseLatestJoinTimeSettings.ChangeListener(__copy); - __copy.addChangeListener(__copy.__changeListener); - return __copy; - } - - private static class ChangeListener - implements com.linkedin.data.collections.CheckedMap.ChangeListener - { - - private final UseLatestJoinTimeSettings __objectRef; - - private ChangeListener(UseLatestJoinTimeSettings reference) { - __objectRef = reference; - } - - @Override - public void onUnderlyingMapChanged(String key, Object value) { - switch (key) { - case "useLatestFeatureData": - __objectRef._useLatestFeatureDataField = null; - break; - } - } - - } - - public static class Fields - extends PathSpec - { - - - public Fields(List path, String name) { - super(path, name); - } - - public Fields() { - super(); - } - - /** - * Boolean value, if set to true, indicates that the latest available feature data is to be used for joining. - * When useLatestFeatureData is set, there should be no other time-based parameters. - * - */ - public PathSpec useLatestFeatureData() { - return new PathSpec(getPathComponents(), "useLatestFeatureData"); - } - - } - - public static class ProjectionMask - extends MaskMap - { - - - ProjectionMask() { - super(2); - } - - /** - * Boolean value, if set to true, indicates that the latest available feature data is to be used for joining. - * When useLatestFeatureData is set, there should be no other time-based parameters. - * - */ - public UseLatestJoinTimeSettings.ProjectionMask withUseLatestFeatureData() { - getDataMap().put("useLatestFeatureData", MaskMap.POSITIVE_MASK); - return this; - } - - } - -}