From 7c25dc65857e300c8efe51407ba69f294d2d9578 Mon Sep 17 00:00:00 2001 From: Aleksey Veresov Date: Fri, 19 Jul 2024 12:49:49 +0200 Subject: [PATCH] Revert "Merge feature-store-api into hsfs subdirectory" This reverts commit d8f3979315512302f963258ba8a1300ecaa40338, reversing changes made to c599739e8bfb1188595674db7ee2fb5dfa420189. --- hsfs/.github/pull_request_template.md | 32 - hsfs/.github/workflows/java-ut.yml | 62 - hsfs/.github/workflows/mkdocs-master.yml | 53 - hsfs/.github/workflows/mkdocs-release.yml | 59 - .../.github/workflows/optional-dependency.yml | 29 - hsfs/.github/workflows/python-lint.yml | 222 - hsfs/.gitignore | 145 - hsfs/CONTRIBUTING.md | 220 - hsfs/Dockerfile | 13 - hsfs/Jenkinsfile | 23 - hsfs/LICENSE | 201 - hsfs/README.md | 201 - hsfs/auto_doc.py | 384 -- hsfs/docs/CONTRIBUTING.md | 220 - hsfs/docs/assets/images/favicon.ico | Bin 2699 -> 0 bytes hsfs/docs/assets/images/hops-logo.png | Bin 6356 -> 0 bytes hsfs/docs/assets/images/hopsworks-logo.png | Bin 20421 -> 0 bytes hsfs/docs/css/custom.css | 114 - hsfs/docs/css/dropdown.css | 55 - hsfs/docs/css/marctech.css | 1047 ---- hsfs/docs/css/version-select.css | 36 - hsfs/docs/index.md | 201 - hsfs/docs/js/dropdown.js | 2 - hsfs/docs/js/inject-api-links.js | 32 - hsfs/docs/js/version-select.js | 64 - hsfs/docs/overrides/main.html | 8 - hsfs/docs/templates/api/connection_api.md | 11 - .../templates/api/embedding_feature_api.md | 7 - .../docs/templates/api/embedding_index_api.md | 12 - hsfs/docs/templates/api/expectation_api.md | 20 - .../templates/api/expectation_suite_api.md | 41 - .../api/external_feature_group_api.md | 19 - hsfs/docs/templates/api/feature_api.md | 11 - .../api/feature_descriptive_statistics_api.md | 7 - hsfs/docs/templates/api/feature_group_api.md | 19 - .../api/feature_monitoring_config_api.md | 27 - .../api/feature_monitoring_result_api.md | 11 - .../feature_monitoring_window_config_api.md | 7 - hsfs/docs/templates/api/feature_store_api.md | 15 - hsfs/docs/templates/api/feature_view_api.md | 21 - hsfs/docs/templates/api/job.md | 11 - hsfs/docs/templates/api/links.md | 14 - hsfs/docs/templates/api/query_api.md | 13 - hsfs/docs/templates/api/rule_api.md | 7 - .../docs/templates/api/rule_definition_api.md | 13 - .../api/similarity_function_type_api.md | 3 - hsfs/docs/templates/api/spine_group_api.md | 19 - .../templates/api/split_statistics_api.md | 7 - hsfs/docs/templates/api/statistics_api.md | 7 - .../templates/api/statistics_config_api.md | 7 - .../templates/api/storage_connector_api.md | 119 - .../templates/api/training_dataset_api.md | 19 - .../api/transformation_functions_api.md | 20 - hsfs/docs/templates/api/validation_api.md | 18 - .../templates/api/validation_report_api.md | 19 - hsfs/java/beam/pom.xml | 55 - .../logicalclocks/hsfs/beam/FeatureStore.java | 322 -- .../logicalclocks/hsfs/beam/FeatureView.java | 152 - .../hsfs/beam/HopsworksConnection.java | 89 - .../hsfs/beam/StreamFeatureGroup.java | 387 -- .../hsfs/beam/constructor/Query.java | 65 - .../hsfs/beam/engine/BeamEngine.java | 100 - .../hsfs/beam/engine/BeamKafkaProducer.java | 54 - .../hsfs/beam/engine/BeamProducer.java | 166 - .../hsfs/beam/engine/FeatureGroupEngine.java | 57 - .../hsfs/beam/engine/FeatureViewEngine.java | 71 - .../beam/engine/GenericAvroSerializer.java | 59 - .../hsfs/beam/engine/KeySerializer.java | 26 - hsfs/java/flink/pom.xml | 92 - .../hsfs/flink/FeatureStore.java | 329 -- .../logicalclocks/hsfs/flink/FeatureView.java | 155 - .../hsfs/flink/HopsworksConnection.java | 88 - .../hsfs/flink/StreamFeatureGroup.java | 389 -- .../hsfs/flink/constructor/FsQuery.java | 39 - .../hsfs/flink/constructor/Query.java | 68 - .../hsfs/flink/engine/FeatureGroupEngine.java | 61 - .../hsfs/flink/engine/FeatureViewEngine.java | 75 - .../hsfs/flink/engine/FlinkEngine.java | 163 - .../flink/engine/KafkaRecordSerializer.java | 97 - .../hsfs/flink/engine/PojoToAvroRecord.java | 141 - hsfs/java/hsfs/pom.xml | 100 - .../com/logicalclocks/hsfs/DataFormat.java | 39 - .../hsfs/DeltaStreamerJobConf.java | 39 - .../hsfs/EntityEndpointType.java | 34 - .../hsfs/ExternalDataFormat.java | 27 - .../java/com/logicalclocks/hsfs/Feature.java | 242 - .../logicalclocks/hsfs/FeatureGroupBase.java | 535 -- .../hsfs/FeatureGroupBaseForApi.java | 309 -- .../hsfs/FeatureGroupCommit.java | 60 - .../logicalclocks/hsfs/FeatureStoreBase.java | 240 - .../hsfs/FeatureStoreException.java | 29 - .../com/logicalclocks/hsfs/FeatureType.java | 24 - .../logicalclocks/hsfs/FeatureViewBase.java | 469 -- .../hsfs/HopsworksConnectionBase.java | 110 - .../logicalclocks/hsfs/HudiOperationType.java | 28 - .../logicalclocks/hsfs/JobConfiguration.java | 69 - .../java/com/logicalclocks/hsfs/Project.java | 43 - .../com/logicalclocks/hsfs/SecretStore.java | 23 - .../logicalclocks/hsfs/SecurityProtocol.java | 33 - .../java/com/logicalclocks/hsfs/Split.java | 72 - .../SslEndpointIdentificationAlgorithm.java | 65 - .../logicalclocks/hsfs/StatisticsConfig.java | 60 - .../java/com/logicalclocks/hsfs/Storage.java | 23 - .../logicalclocks/hsfs/StorageConnector.java | 562 -- .../hsfs/StorageConnectorType.java | 30 - .../logicalclocks/hsfs/TimeTravelFormat.java | 23 - .../hsfs/TrainingDatasetBase.java | 421 -- .../hsfs/TrainingDatasetFeature.java | 85 - .../hsfs/TrainingDatasetType.java | 24 - .../hsfs/TransformationFunction.java | 51 - .../hsfs/constructor/FeatureGroupAlias.java | 49 - .../hsfs/constructor/Filter.java | 54 - .../hsfs/constructor/FilterLogic.java | 86 - .../hsfs/constructor/FsQueryBase.java | 74 - .../logicalclocks/hsfs/constructor/Join.java | 72 - .../hsfs/constructor/JoinType.java | 28 - .../PreparedStatementParameter.java | 34 - .../hsfs/constructor/QueryBase.java | 317 -- .../constructor/ServingPreparedStatement.java | 44 - .../hsfs/constructor/SqlFilterCondition.java | 29 - .../hsfs/constructor/SqlFilterLogic.java | 24 - .../logicalclocks/hsfs/engine/CodeEngine.java | 102 - .../logicalclocks/hsfs/engine/EngineBase.java | 39 - .../hsfs/engine/FeatureGroupEngineBase.java | 134 - .../hsfs/engine/FeatureGroupUtils.java | 250 - .../hsfs/engine/FeatureViewEngineBase.java | 311 -- .../hsfs/engine/VectorServer.java | 444 -- .../hsfs/metadata/AuthorizationHandler.java | 57 - .../com/logicalclocks/hsfs/metadata/Code.java | 57 - .../logicalclocks/hsfs/metadata/CodeApi.java | 80 - .../hsfs/metadata/Credentials.java | 54 - .../hsfs/metadata/DatasetApi.java | 58 - .../FeatureDescriptiveStatistics.java | 147 - .../hsfs/metadata/FeatureGroupApi.java | 307 -- .../hsfs/metadata/FeatureStoreApi.java | 50 - .../hsfs/metadata/FeatureViewApi.java | 355 -- .../hsfs/metadata/HopsworksClient.java | 214 - .../metadata/HopsworksExternalClient.java | 259 - .../metadata/HopsworksHostnameVerifier.java | 37 - .../hsfs/metadata/HopsworksHttpClient.java | 55 - .../metadata/HopsworksInternalClient.java | 187 - .../hsfs/metadata/InternalException.java | 26 - .../logicalclocks/hsfs/metadata/KafkaApi.java | 58 - .../hsfs/metadata/KafkaClusterInfo.java | 36 - .../hsfs/metadata/OnDemandOptions.java | 37 - .../logicalclocks/hsfs/metadata/Option.java | 35 - .../hsfs/metadata/ProjectApi.java | 43 - .../hsfs/metadata/QueryConstructorApi.java | 76 - .../logicalclocks/hsfs/metadata/RestDto.java | 52 - .../hsfs/metadata/SplitStatistics.java | 36 - .../hsfs/metadata/Statistics.java | 67 - .../hsfs/metadata/StatisticsApi.java | 177 - .../hsfs/metadata/StorageConnectorApi.java | 95 - .../logicalclocks/hsfs/metadata/Subject.java | 46 - .../com/logicalclocks/hsfs/metadata/Tags.java | 32 - .../logicalclocks/hsfs/metadata/TagsApi.java | 281 - .../hsfs/metadata/TrainingDatasetApi.java | 220 - .../TransformationFunctionAttached.java | 40 - .../hsfs/metadata/UnauthorizedException.java | 23 - .../com/logicalclocks/hsfs/metadata/User.java | 41 - .../logicalclocks/hsfs/metadata/Variable.java | 33 - .../hsfs/metadata/VariablesApi.java | 51 - .../logicalclocks/hsfs/util/Constants.java | 104 - .../hsfs/TestFeatureGroupBaseForApi.java | 75 - .../hsfs/TestHopsworksExternalClient.java | 69 - .../hsfs/engine/TestFeatureGroupUtils.java | 76 - .../hsfs/metadata/TestHopsworksClient.java | 49 - .../hsfs/metadata/TestTagsApi.java | 85 - hsfs/java/pom.xml | 308 -- hsfs/java/spark/pom.xml | 165 - .../hsfs/spark/ExternalFeatureGroup.java | 476 -- .../hsfs/spark/FeatureGroup.java | 1271 ----- .../hsfs/spark/FeatureStore.java | 1037 ---- .../logicalclocks/hsfs/spark/FeatureView.java | 1914 ------- .../hsfs/spark/HopsworksConnection.java | 91 - .../logicalclocks/hsfs/spark/MainClass.java | 76 - .../hsfs/spark/StreamFeatureGroup.java | 1405 ----- .../hsfs/spark/TrainingDataset.java | 346 -- .../hsfs/spark/TrainingDatasetBundle.java | 88 - .../hsfs/spark/constructor/FsQuery.java | 56 - .../hsfs/spark/constructor/Query.java | 96 - .../hsfs/spark/engine/FeatureGroupEngine.java | 510 -- .../hsfs/spark/engine/FeatureViewEngine.java | 298 -- .../hsfs/spark/engine/SparkEngine.java | 1106 ---- .../hsfs/spark/engine/StatisticsEngine.java | 193 - .../spark/engine/TrainingDatasetEngine.java | 145 - .../spark/engine/TrainingDatasetUtils.java | 81 - .../hudi/DeltaStreamerAvroDeserializer.java | 169 - .../engine/hudi/DeltaStreamerConfig.java | 149 - .../engine/hudi/DeltaStreamerKafkaSource.java | 115 - .../hudi/DeltaStreamerSchemaProvider.java | 49 - .../engine/hudi/DeltaStreamerTransformer.java | 40 - .../hsfs/spark/engine/hudi/HudiEngine.java | 411 -- .../spark/util/StorageConnectorUtils.java | 294 -- .../hsfs/spark/TestExternalFeatureGroup.java | 89 - .../logicalclocks/hsfs/spark/TestFeature.java | 41 - .../hsfs/spark/TestFeatureGroup.java | 176 - .../hsfs/spark/TestFeatureView.java | 93 - .../hsfs/spark/TestStorageConnector.java | 277 - .../hsfs/spark/constructor/TestQuery.java | 72 - .../spark/engine/TestFeatureViewEngine.java | 180 - .../hsfs/spark/engine/TestHudiEngine.java | 71 - .../hsfs/spark/engine/TestSparkEngine.java | 256 - .../test/resources/hadoop/bin/winutils.exe | Bin 112640 -> 0 bytes .../src/test/resources/system.properties | 1 - hsfs/java/src/main/resources/checkstyle.xml | 312 -- hsfs/java/src/main/resources/suppressions.xml | 8 - .../test/resources/hadoop/bin/winutils.exe | Bin 112640 -> 0 bytes .../java/src/test/resources/system.properties | 1 - hsfs/locust_benchmark/Dockerfile | 11 - hsfs/locust_benchmark/README.md | 179 - hsfs/locust_benchmark/common/__init__.py | 0 .../common/hopsworks_client.py | 100 - hsfs/locust_benchmark/common/stop_watch.py | 35 - hsfs/locust_benchmark/create_feature_group.py | 8 - hsfs/locust_benchmark/docker-compose.yml | 21 - hsfs/locust_benchmark/hopsworks_config.json | 10 - hsfs/locust_benchmark/locustfile.py | 120 - hsfs/locust_benchmark/requirements.txt | 3 - hsfs/mkdocs.yml | 130 - hsfs/python/.pre-commit-config.yaml | 8 - hsfs/python/hsfs/__init__.py | 59 - hsfs/python/hsfs/builtin_transformations.py | 66 - hsfs/python/hsfs/client/__init__.py | 70 - hsfs/python/hsfs/client/auth.py | 51 - hsfs/python/hsfs/client/base.py | 289 - hsfs/python/hsfs/client/exceptions.py | 110 - hsfs/python/hsfs/client/external.py | 382 -- hsfs/python/hsfs/client/hopsworks.py | 185 - .../hsfs/client/online_store_rest_client.py | 384 -- hsfs/python/hsfs/code.py | 63 - hsfs/python/hsfs/connection.py | 416 -- hsfs/python/hsfs/constructor/__init__.py | 15 - .../external_feature_group_alias.py | 56 - hsfs/python/hsfs/constructor/filter.py | 214 - hsfs/python/hsfs/constructor/fs_query.py | 133 - .../constructor/hudi_feature_group_alias.py | 59 - hsfs/python/hsfs/constructor/join.py | 86 - .../prepared_statement_parameter.py | 73 - hsfs/python/hsfs/constructor/query.py | 836 --- .../constructor/serving_prepared_statement.py | 140 - hsfs/python/hsfs/core/__init__.py | 15 - hsfs/python/hsfs/core/arrow_flight_client.py | 646 --- hsfs/python/hsfs/core/code_api.py | 62 - hsfs/python/hsfs/core/code_engine.py | 69 - hsfs/python/hsfs/core/constants.py | 35 - hsfs/python/hsfs/core/dataset_api.py | 110 - hsfs/python/hsfs/core/delta_engine.py | 204 - .../python/hsfs/core/deltastreamer_jobconf.py | 35 - hsfs/python/hsfs/core/execution.py | 64 - hsfs/python/hsfs/core/expectation_api.py | 161 - hsfs/python/hsfs/core/expectation_engine.py | 71 - .../python/hsfs/core/expectation_suite_api.py | 185 - .../hsfs/core/expectation_suite_engine.py | 110 - hsfs/python/hsfs/core/explicit_provenance.py | 444 -- .../core/external_feature_group_engine.py | 149 - .../core/feature_descriptive_statistics.py | 319 -- hsfs/python/hsfs/core/feature_group_api.py | 542 -- .../hsfs/core/feature_group_base_engine.py | 182 - hsfs/python/hsfs/core/feature_group_engine.py | 390 -- hsfs/python/hsfs/core/feature_logging.py | 53 - .../hsfs/core/feature_monitoring_config.py | 848 --- .../core/feature_monitoring_config_api.py | 281 - .../core/feature_monitoring_config_engine.py | 512 -- .../hsfs/core/feature_monitoring_result.py | 210 - .../core/feature_monitoring_result_api.py | 166 - .../core/feature_monitoring_result_engine.py | 585 -- hsfs/python/hsfs/core/feature_store_api.py | 37 - hsfs/python/hsfs/core/feature_view_api.py | 500 -- hsfs/python/hsfs/core/feature_view_engine.py | 1116 ---- .../hsfs/core/great_expectation_engine.py | 167 - hsfs/python/hsfs/core/hosts_api.py | 29 - hsfs/python/hsfs/core/hudi_engine.py | 274 - hsfs/python/hsfs/core/ingestion_job.py | 48 - hsfs/python/hsfs/core/ingestion_job_conf.py | 77 - hsfs/python/hsfs/core/inode.py | 42 - hsfs/python/hsfs/core/job.py | 253 - hsfs/python/hsfs/core/job_api.py | 95 - hsfs/python/hsfs/core/job_configuration.py | 61 - hsfs/python/hsfs/core/job_schedule.py | 107 - hsfs/python/hsfs/core/kafka_api.py | 43 - hsfs/python/hsfs/core/kafka_engine.py | 248 - .../hsfs/core/monitoring_window_config.py | 301 -- .../core/monitoring_window_config_engine.py | 436 -- .../hsfs/core/online_store_rest_client_api.py | 187 - .../core/online_store_rest_client_engine.py | 438 -- .../hsfs/core/online_store_sql_engine.py | 741 --- hsfs/python/hsfs/core/opensearch.py | 224 - hsfs/python/hsfs/core/opensearch_api.py | 129 - hsfs/python/hsfs/core/project_api.py | 29 - .../python/hsfs/core/query_constructor_api.py | 31 - hsfs/python/hsfs/core/services_api.py | 30 - hsfs/python/hsfs/core/spine_group_engine.py | 47 - hsfs/python/hsfs/core/statistics_api.py | 341 -- hsfs/python/hsfs/core/statistics_engine.py | 449 -- .../python/hsfs/core/storage_connector_api.py | 142 - hsfs/python/hsfs/core/tags_api.py | 132 - hsfs/python/hsfs/core/training_dataset_api.py | 207 - .../hsfs/core/training_dataset_engine.py | 160 - .../hsfs/core/training_dataset_job_conf.py | 75 - .../hsfs/core/transformation_function_api.py | 112 - .../core/transformation_function_engine.py | 301 -- hsfs/python/hsfs/core/util_sql.py | 117 - .../python/hsfs/core/validation_report_api.py | 141 - .../hsfs/core/validation_report_engine.py | 112 - .../python/hsfs/core/validation_result_api.py | 64 - .../hsfs/core/validation_result_engine.py | 138 - hsfs/python/hsfs/core/variable_api.py | 78 - hsfs/python/hsfs/core/vector_db_client.py | 436 -- hsfs/python/hsfs/core/vector_server.py | 1150 ---- hsfs/python/hsfs/decorators.py | 85 - hsfs/python/hsfs/embedding.py | 398 -- hsfs/python/hsfs/engine/__init__.py | 93 - hsfs/python/hsfs/engine/python.py | 1736 ------ hsfs/python/hsfs/engine/spark.py | 1474 ------ hsfs/python/hsfs/engine/spark_no_metastore.py | 37 - hsfs/python/hsfs/expectation_suite.py | 619 --- hsfs/python/hsfs/feature.py | 263 - hsfs/python/hsfs/feature_group.py | 4213 --------------- hsfs/python/hsfs/feature_group_commit.py | 140 - hsfs/python/hsfs/feature_group_writer.py | 63 - hsfs/python/hsfs/feature_store.py | 1765 ------- hsfs/python/hsfs/feature_view.py | 3944 -------------- hsfs/python/hsfs/ge_expectation.py | 170 - hsfs/python/hsfs/ge_validation_result.py | 267 - hsfs/python/hsfs/hopsworks_udf.py | 715 --- hsfs/python/hsfs/serving_key.py | 121 - hsfs/python/hsfs/split_statistics.py | 72 - hsfs/python/hsfs/statistics.py | 233 - hsfs/python/hsfs/statistics_config.py | 115 - hsfs/python/hsfs/storage_connector.py | 1686 ------ hsfs/python/hsfs/tag.py | 78 - hsfs/python/hsfs/training_dataset.py | 1111 ---- hsfs/python/hsfs/training_dataset_feature.py | 131 - hsfs/python/hsfs/training_dataset_split.py | 109 - hsfs/python/hsfs/transformation_function.py | 235 - hsfs/python/hsfs/transformation_statistics.py | 124 - hsfs/python/hsfs/usage.py | 279 - hsfs/python/hsfs/user.py | 135 - hsfs/python/hsfs/util.py | 496 -- hsfs/python/hsfs/validation_report.py | 259 - hsfs/python/hsfs/version.py | 17 - hsfs/python/pyproject.toml | 173 - hsfs/python/setup.py | 4 - hsfs/python/tests/__init__.py | 15 - hsfs/python/tests/client/test_base_client.py | 129 - hsfs/python/tests/conftest.py | 32 - .../test_external_feature_group_alias.py | 39 - hsfs/python/tests/constructor/test_filter.py | 49 - .../python/tests/constructor/test_fs_query.py | 60 - .../test_hudi_feature_group_alias.py | 55 - hsfs/python/tests/constructor/test_join.py | 77 - .../test_prepared_statement_parameter.py | 52 - hsfs/python/tests/constructor/test_query.py | 398 -- .../test_serving_prepared_statement.py | 56 - hsfs/python/tests/core/__init__.py | 15 - .../tests/core/test_arrow_flight_client.py | 536 -- hsfs/python/tests/core/test_code_engine.py | 146 - hsfs/python/tests/core/test_execution.py | 44 - .../tests/core/test_expectation_engine.py | 141 - .../core/test_expectation_suite_engine.py | 189 - .../test_external_feature_group_engine.py | 387 -- .../test_feature_descriptive_statistics.py | 331 -- .../tests/core/test_feature_group_api.py | 82 - .../core/test_feature_group_base_engine.py | 150 - .../tests/core/test_feature_group_engine.py | 1358 ----- .../core/test_feature_monitoring_config.py | 283 - .../test_feature_monitoring_config_engine.py | 107 - .../core/test_feature_monitoring_result.py | 231 - .../test_feature_monitoring_result_engine.py | 790 --- .../tests/core/test_feature_view_engine.py | 2224 -------- .../core/test_great_expectation_engine.py | 620 --- hsfs/python/tests/core/test_hudi_engine.py | 381 -- hsfs/python/tests/core/test_ingestion_job.py | 31 - hsfs/python/tests/core/test_inode.py | 42 - hsfs/python/tests/core/test_job.py | 132 - .../tests/core/test_job_configuration.py | 71 - hsfs/python/tests/core/test_kafka_engine.py | 527 -- .../core/test_monitoring_window_config.py | 135 - .../test_monitoring_window_config_engine.py | 385 -- .../core/test_online_store_rest_client.py | 119 - .../core/test_online_store_rest_client_api.py | 84 - .../test_online_store_rest_client_engine.py | 409 -- hsfs/python/tests/core/test_opensearch.py | 91 - .../tests/core/test_statistics_engine.py | 915 ---- .../core/test_training_dataset_engine.py | 470 -- .../test_transformation_function_engine.py | 477 -- .../core/test_validation_report_engine.py | 164 - .../core/test_validation_result_engine.py | 191 - .../tests/core/test_vector_db_client.py | 251 - .../python/tests/data/hadoop/bin/winutils.exe | Bin 112640 -> 0 bytes hsfs/python/tests/data/test_basic.csv | 5 - hsfs/python/tests/data/test_basic.parquet | Bin 3647 -> 0 bytes hsfs/python/tests/data/test_basic.tsv | 5 - hsfs/python/tests/engine/__init__.py | 15 - hsfs/python/tests/engine/test_python.py | 3641 ------------- .../python/tests/engine/test_python_reader.py | 170 - .../test_python_spark_convert_dataframe.py | 128 - ...t_python_spark_transformation_functions.py | 1099 ---- .../python/tests/engine/test_python_writer.py | 114 - hsfs/python/tests/engine/test_spark.py | 4695 ----------------- hsfs/python/tests/fixtures/__init__.py | 15 - .../python/tests/fixtures/backend_fixtures.py | 75 - .../tests/fixtures/dataframe_fixtures.py | 75 - .../tests/fixtures/execution_fixtures.json | 34 - .../fixtures/expectation_suite_fixtures.json | 79 - ...external_feature_group_alias_fixtures.json | 19 - .../external_feature_group_fixtures.json | 254 - ...ature_descriptive_statistics_fixtures.json | 652 --- .../tests/fixtures/feature_fixtures.json | 21 - .../feature_group_commit_fixtures.json | 27 - .../fixtures/feature_group_fixtures.json | 634 --- .../feature_monitoring_config_fixtures.json | 310 -- .../feature_monitoring_result_fixtures.json | 359 -- .../fixtures/feature_store_fixtures.json | 30 - .../tests/fixtures/feature_view_fixtures.json | 996 ---- .../tests/fixtures/filter_fixtures.json | 18 - .../tests/fixtures/fs_query_fixtures.json | 51 - .../fixtures/ge_expectation_fixtures.json | 29 - .../ge_validation_result_fixtures.json | 58 - .../fixtures/generate_backend_fixtures.ipynb | 576 -- .../hudi_feature_group_alias_fixtures.json | 28 - .../fixtures/ingestion_job_fixtures.json | 26 - .../python/tests/fixtures/inode_fixtures.json | 25 - hsfs/python/tests/fixtures/job_fixtures.json | 28 - hsfs/python/tests/fixtures/join_fixtures.json | 235 - .../python/tests/fixtures/logic_fixtures.json | 41 - ...prepared_statement_parameter_fixtures.json | 25 - .../python/tests/fixtures/query_fixtures.json | 479 -- .../tests/fixtures/rondb_server_fixtures.json | 291 - .../tests/fixtures/serving_keys_fixtures.json | 120 - .../serving_prepared_statement_fixtures.json | 147 - .../tests/fixtures/spine_group_fixtures.json | 58 - .../fixtures/split_statistics_fixtures.json | 69 - .../fixtures/statistics_config_fixtures.json | 14 - .../tests/fixtures/statistics_fixtures.json | 287 - .../fixtures/storage_connector_fixtures.json | 550 -- hsfs/python/tests/fixtures/tag_fixtures.json | 25 - .../training_dataset_feature_fixtures.json | 450 -- .../fixtures/training_dataset_fixtures.json | 203 - .../training_dataset_split_fixtures.json | 17 - ...sformation_function_attached_fixtures.json | 68 - .../transformation_function_fixtures.json | 112 - hsfs/python/tests/fixtures/user_fixtures.json | 25 - .../fixtures/validation_report_fixtures.json | 105 - hsfs/python/tests/pyproject.toml | 37 - hsfs/python/tests/test_expectation_suite.py | 92 - hsfs/python/tests/test_feature.py | 84 - hsfs/python/tests/test_feature_group.py | 891 ---- .../python/tests/test_feature_group_commit.py | 66 - .../python/tests/test_feature_group_writer.py | 162 - hsfs/python/tests/test_feature_store.py | 109 - hsfs/python/tests/test_feature_view.py | 161 - hsfs/python/tests/test_ge_expectation.py | 89 - .../python/tests/test_ge_validation_result.py | 104 - hsfs/python/tests/test_helpers/__init__.py | 0 .../transformation_test_helper.py | 81 - hsfs/python/tests/test_hopswork_udf.py | 473 -- hsfs/python/tests/test_serving_keys.py | 27 - hsfs/python/tests/test_split_statistics.py | 63 - hsfs/python/tests/test_statistics.py | 248 - hsfs/python/tests/test_statistics_config.py | 48 - hsfs/python/tests/test_storage_connector.py | 907 ---- hsfs/python/tests/test_tag.py | 43 - hsfs/python/tests/test_training_dataset.py | 123 - .../tests/test_training_dataset_feature.py | 58 - .../tests/test_training_dataset_split.py | 48 - .../tests/test_transformation_function.py | 230 - hsfs/python/tests/test_user.py | 55 - hsfs/python/tests/test_util.py | 230 - hsfs/python/tests/test_validation_report.py | 99 - hsfs/python/tests/util.py | 31 - hsfs/requirements-docs.txt | 12 - hsfs/utils/java/pom.xml | 129 - .../com/logicalclocks/utils/MainClass.java | 126 - .../java/src/main/resources/checkstyle.xml | 312 -- .../java/src/main/resources/suppressions.xml | 8 - hsfs/utils/python/hsfs_utils.py | 297 -- 478 files changed, 108044 deletions(-) delete mode 100644 hsfs/.github/pull_request_template.md delete mode 100644 hsfs/.github/workflows/java-ut.yml delete mode 100644 hsfs/.github/workflows/mkdocs-master.yml delete mode 100644 hsfs/.github/workflows/mkdocs-release.yml delete mode 100644 hsfs/.github/workflows/optional-dependency.yml delete mode 100644 hsfs/.github/workflows/python-lint.yml delete mode 100644 hsfs/.gitignore delete mode 100644 hsfs/CONTRIBUTING.md delete mode 100644 hsfs/Dockerfile delete mode 100644 hsfs/Jenkinsfile delete mode 100644 hsfs/LICENSE delete mode 100644 hsfs/README.md delete mode 100644 hsfs/auto_doc.py delete mode 100644 hsfs/docs/CONTRIBUTING.md delete mode 100644 hsfs/docs/assets/images/favicon.ico delete mode 100644 hsfs/docs/assets/images/hops-logo.png delete mode 100644 hsfs/docs/assets/images/hopsworks-logo.png delete mode 100644 hsfs/docs/css/custom.css delete mode 100644 hsfs/docs/css/dropdown.css delete mode 100644 hsfs/docs/css/marctech.css delete mode 100644 hsfs/docs/css/version-select.css delete mode 100644 hsfs/docs/index.md delete mode 100644 hsfs/docs/js/dropdown.js delete mode 100644 hsfs/docs/js/inject-api-links.js delete mode 100644 hsfs/docs/js/version-select.js delete mode 100644 hsfs/docs/overrides/main.html delete mode 100644 hsfs/docs/templates/api/connection_api.md delete mode 100644 hsfs/docs/templates/api/embedding_feature_api.md delete mode 100644 hsfs/docs/templates/api/embedding_index_api.md delete mode 100644 hsfs/docs/templates/api/expectation_api.md delete mode 100644 hsfs/docs/templates/api/expectation_suite_api.md delete mode 100644 hsfs/docs/templates/api/external_feature_group_api.md delete mode 100644 hsfs/docs/templates/api/feature_api.md delete mode 100644 hsfs/docs/templates/api/feature_descriptive_statistics_api.md delete mode 100644 hsfs/docs/templates/api/feature_group_api.md delete mode 100644 hsfs/docs/templates/api/feature_monitoring_config_api.md delete mode 100644 hsfs/docs/templates/api/feature_monitoring_result_api.md delete mode 100644 hsfs/docs/templates/api/feature_monitoring_window_config_api.md delete mode 100644 hsfs/docs/templates/api/feature_store_api.md delete mode 100644 hsfs/docs/templates/api/feature_view_api.md delete mode 100644 hsfs/docs/templates/api/job.md delete mode 100644 hsfs/docs/templates/api/links.md delete mode 100644 hsfs/docs/templates/api/query_api.md delete mode 100644 hsfs/docs/templates/api/rule_api.md delete mode 100644 hsfs/docs/templates/api/rule_definition_api.md delete mode 100644 hsfs/docs/templates/api/similarity_function_type_api.md delete mode 100644 hsfs/docs/templates/api/spine_group_api.md delete mode 100644 hsfs/docs/templates/api/split_statistics_api.md delete mode 100644 hsfs/docs/templates/api/statistics_api.md delete mode 100644 hsfs/docs/templates/api/statistics_config_api.md delete mode 100644 hsfs/docs/templates/api/storage_connector_api.md delete mode 100644 hsfs/docs/templates/api/training_dataset_api.md delete mode 100644 hsfs/docs/templates/api/transformation_functions_api.md delete mode 100644 hsfs/docs/templates/api/validation_api.md delete mode 100644 hsfs/docs/templates/api/validation_report_api.md delete mode 100644 hsfs/java/beam/pom.xml delete mode 100644 hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/FeatureStore.java delete mode 100644 hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/FeatureView.java delete mode 100644 hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/HopsworksConnection.java delete mode 100644 hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/StreamFeatureGroup.java delete mode 100644 hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/constructor/Query.java delete mode 100644 hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/BeamEngine.java delete mode 100644 hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/BeamKafkaProducer.java delete mode 100644 hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/BeamProducer.java delete mode 100644 hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/FeatureGroupEngine.java delete mode 100644 hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/FeatureViewEngine.java delete mode 100644 hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/GenericAvroSerializer.java delete mode 100644 hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/KeySerializer.java delete mode 100644 hsfs/java/flink/pom.xml delete mode 100644 hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/FeatureStore.java delete mode 100644 hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/FeatureView.java delete mode 100644 hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/HopsworksConnection.java delete mode 100644 hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/StreamFeatureGroup.java delete mode 100644 hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/constructor/FsQuery.java delete mode 100644 hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/constructor/Query.java delete mode 100644 hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/FeatureGroupEngine.java delete mode 100644 hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/FeatureViewEngine.java delete mode 100644 hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/FlinkEngine.java delete mode 100644 hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/KafkaRecordSerializer.java delete mode 100644 hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/PojoToAvroRecord.java delete mode 100644 hsfs/java/hsfs/pom.xml delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/DataFormat.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/ExternalDataFormat.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/Feature.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupBase.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupBaseForApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupCommit.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureStoreBase.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureType.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureViewBase.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/HopsworksConnectionBase.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/HudiOperationType.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/JobConfiguration.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/Project.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/SecretStore.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/SecurityProtocol.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/Split.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/SslEndpointIdentificationAlgorithm.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/StatisticsConfig.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/Storage.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/StorageConnector.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/TimeTravelFormat.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetBase.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetFeature.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/TransformationFunction.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FeatureGroupAlias.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/Filter.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FilterLogic.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FsQueryBase.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/Join.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/JoinType.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/PreparedStatementParameter.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/QueryBase.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/ServingPreparedStatement.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterCondition.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterLogic.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/CodeEngine.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/EngineBase.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/VectorServer.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Code.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/CodeApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/DatasetApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureDescriptiveStatistics.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureViewApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/KafkaApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/KafkaClusterInfo.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/OnDemandOptions.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Option.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/RestDto.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/SplitStatistics.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Statistics.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Subject.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TransformationFunctionAttached.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/User.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Variable.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/VariablesApi.java delete mode 100644 hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/util/Constants.java delete mode 100644 hsfs/java/hsfs/src/test/java/com/logicalclocks/hsfs/TestFeatureGroupBaseForApi.java delete mode 100644 hsfs/java/hsfs/src/test/java/com/logicalclocks/hsfs/TestHopsworksExternalClient.java delete mode 100644 hsfs/java/hsfs/src/test/java/com/logicalclocks/hsfs/engine/TestFeatureGroupUtils.java delete mode 100644 hsfs/java/hsfs/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java delete mode 100644 hsfs/java/hsfs/src/test/java/com/logicalclocks/hsfs/metadata/TestTagsApi.java delete mode 100644 hsfs/java/pom.xml delete mode 100644 hsfs/java/spark/pom.xml delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/TrainingDatasetBundle.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/constructor/FsQuery.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/constructor/Query.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/FeatureGroupEngine.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/FeatureViewEngine.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/SparkEngine.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/StatisticsEngine.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetEngine.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetUtils.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerAvroDeserializer.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerConfig.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerKafkaSource.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerSchemaProvider.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerTransformer.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/HudiEngine.java delete mode 100644 hsfs/java/spark/src/main/java/com/logicalclocks/hsfs/spark/util/StorageConnectorUtils.java delete mode 100644 hsfs/java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestExternalFeatureGroup.java delete mode 100644 hsfs/java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestFeature.java delete mode 100644 hsfs/java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestFeatureGroup.java delete mode 100644 hsfs/java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestFeatureView.java delete mode 100644 hsfs/java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestStorageConnector.java delete mode 100644 hsfs/java/spark/src/test/java/com/logicalclocks/hsfs/spark/constructor/TestQuery.java delete mode 100644 hsfs/java/spark/src/test/java/com/logicalclocks/hsfs/spark/engine/TestFeatureViewEngine.java delete mode 100644 hsfs/java/spark/src/test/java/com/logicalclocks/hsfs/spark/engine/TestHudiEngine.java delete mode 100644 hsfs/java/spark/src/test/java/com/logicalclocks/hsfs/spark/engine/TestSparkEngine.java delete mode 100644 hsfs/java/spark/src/test/resources/hadoop/bin/winutils.exe delete mode 100644 hsfs/java/spark/src/test/resources/system.properties delete mode 100644 hsfs/java/src/main/resources/checkstyle.xml delete mode 100644 hsfs/java/src/main/resources/suppressions.xml delete mode 100644 hsfs/java/src/test/resources/hadoop/bin/winutils.exe delete mode 100644 hsfs/java/src/test/resources/system.properties delete mode 100644 hsfs/locust_benchmark/Dockerfile delete mode 100644 hsfs/locust_benchmark/README.md delete mode 100644 hsfs/locust_benchmark/common/__init__.py delete mode 100644 hsfs/locust_benchmark/common/hopsworks_client.py delete mode 100644 hsfs/locust_benchmark/common/stop_watch.py delete mode 100644 hsfs/locust_benchmark/create_feature_group.py delete mode 100644 hsfs/locust_benchmark/docker-compose.yml delete mode 100644 hsfs/locust_benchmark/hopsworks_config.json delete mode 100644 hsfs/locust_benchmark/locustfile.py delete mode 100644 hsfs/locust_benchmark/requirements.txt delete mode 100644 hsfs/mkdocs.yml delete mode 100644 hsfs/python/.pre-commit-config.yaml delete mode 100644 hsfs/python/hsfs/__init__.py delete mode 100644 hsfs/python/hsfs/builtin_transformations.py delete mode 100644 hsfs/python/hsfs/client/__init__.py delete mode 100644 hsfs/python/hsfs/client/auth.py delete mode 100644 hsfs/python/hsfs/client/base.py delete mode 100644 hsfs/python/hsfs/client/exceptions.py delete mode 100644 hsfs/python/hsfs/client/external.py delete mode 100644 hsfs/python/hsfs/client/hopsworks.py delete mode 100644 hsfs/python/hsfs/client/online_store_rest_client.py delete mode 100644 hsfs/python/hsfs/code.py delete mode 100644 hsfs/python/hsfs/connection.py delete mode 100644 hsfs/python/hsfs/constructor/__init__.py delete mode 100644 hsfs/python/hsfs/constructor/external_feature_group_alias.py delete mode 100644 hsfs/python/hsfs/constructor/filter.py delete mode 100644 hsfs/python/hsfs/constructor/fs_query.py delete mode 100644 hsfs/python/hsfs/constructor/hudi_feature_group_alias.py delete mode 100644 hsfs/python/hsfs/constructor/join.py delete mode 100644 hsfs/python/hsfs/constructor/prepared_statement_parameter.py delete mode 100644 hsfs/python/hsfs/constructor/query.py delete mode 100644 hsfs/python/hsfs/constructor/serving_prepared_statement.py delete mode 100644 hsfs/python/hsfs/core/__init__.py delete mode 100644 hsfs/python/hsfs/core/arrow_flight_client.py delete mode 100644 hsfs/python/hsfs/core/code_api.py delete mode 100644 hsfs/python/hsfs/core/code_engine.py delete mode 100644 hsfs/python/hsfs/core/constants.py delete mode 100644 hsfs/python/hsfs/core/dataset_api.py delete mode 100644 hsfs/python/hsfs/core/delta_engine.py delete mode 100644 hsfs/python/hsfs/core/deltastreamer_jobconf.py delete mode 100644 hsfs/python/hsfs/core/execution.py delete mode 100644 hsfs/python/hsfs/core/expectation_api.py delete mode 100644 hsfs/python/hsfs/core/expectation_engine.py delete mode 100644 hsfs/python/hsfs/core/expectation_suite_api.py delete mode 100644 hsfs/python/hsfs/core/expectation_suite_engine.py delete mode 100644 hsfs/python/hsfs/core/explicit_provenance.py delete mode 100644 hsfs/python/hsfs/core/external_feature_group_engine.py delete mode 100644 hsfs/python/hsfs/core/feature_descriptive_statistics.py delete mode 100644 hsfs/python/hsfs/core/feature_group_api.py delete mode 100644 hsfs/python/hsfs/core/feature_group_base_engine.py delete mode 100644 hsfs/python/hsfs/core/feature_group_engine.py delete mode 100644 hsfs/python/hsfs/core/feature_logging.py delete mode 100644 hsfs/python/hsfs/core/feature_monitoring_config.py delete mode 100644 hsfs/python/hsfs/core/feature_monitoring_config_api.py delete mode 100644 hsfs/python/hsfs/core/feature_monitoring_config_engine.py delete mode 100644 hsfs/python/hsfs/core/feature_monitoring_result.py delete mode 100644 hsfs/python/hsfs/core/feature_monitoring_result_api.py delete mode 100644 hsfs/python/hsfs/core/feature_monitoring_result_engine.py delete mode 100644 hsfs/python/hsfs/core/feature_store_api.py delete mode 100644 hsfs/python/hsfs/core/feature_view_api.py delete mode 100644 hsfs/python/hsfs/core/feature_view_engine.py delete mode 100644 hsfs/python/hsfs/core/great_expectation_engine.py delete mode 100644 hsfs/python/hsfs/core/hosts_api.py delete mode 100644 hsfs/python/hsfs/core/hudi_engine.py delete mode 100644 hsfs/python/hsfs/core/ingestion_job.py delete mode 100644 hsfs/python/hsfs/core/ingestion_job_conf.py delete mode 100644 hsfs/python/hsfs/core/inode.py delete mode 100644 hsfs/python/hsfs/core/job.py delete mode 100644 hsfs/python/hsfs/core/job_api.py delete mode 100644 hsfs/python/hsfs/core/job_configuration.py delete mode 100644 hsfs/python/hsfs/core/job_schedule.py delete mode 100644 hsfs/python/hsfs/core/kafka_api.py delete mode 100644 hsfs/python/hsfs/core/kafka_engine.py delete mode 100644 hsfs/python/hsfs/core/monitoring_window_config.py delete mode 100644 hsfs/python/hsfs/core/monitoring_window_config_engine.py delete mode 100644 hsfs/python/hsfs/core/online_store_rest_client_api.py delete mode 100644 hsfs/python/hsfs/core/online_store_rest_client_engine.py delete mode 100644 hsfs/python/hsfs/core/online_store_sql_engine.py delete mode 100644 hsfs/python/hsfs/core/opensearch.py delete mode 100644 hsfs/python/hsfs/core/opensearch_api.py delete mode 100644 hsfs/python/hsfs/core/project_api.py delete mode 100644 hsfs/python/hsfs/core/query_constructor_api.py delete mode 100644 hsfs/python/hsfs/core/services_api.py delete mode 100644 hsfs/python/hsfs/core/spine_group_engine.py delete mode 100644 hsfs/python/hsfs/core/statistics_api.py delete mode 100644 hsfs/python/hsfs/core/statistics_engine.py delete mode 100644 hsfs/python/hsfs/core/storage_connector_api.py delete mode 100644 hsfs/python/hsfs/core/tags_api.py delete mode 100644 hsfs/python/hsfs/core/training_dataset_api.py delete mode 100644 hsfs/python/hsfs/core/training_dataset_engine.py delete mode 100644 hsfs/python/hsfs/core/training_dataset_job_conf.py delete mode 100644 hsfs/python/hsfs/core/transformation_function_api.py delete mode 100644 hsfs/python/hsfs/core/transformation_function_engine.py delete mode 100644 hsfs/python/hsfs/core/util_sql.py delete mode 100644 hsfs/python/hsfs/core/validation_report_api.py delete mode 100644 hsfs/python/hsfs/core/validation_report_engine.py delete mode 100644 hsfs/python/hsfs/core/validation_result_api.py delete mode 100644 hsfs/python/hsfs/core/validation_result_engine.py delete mode 100644 hsfs/python/hsfs/core/variable_api.py delete mode 100644 hsfs/python/hsfs/core/vector_db_client.py delete mode 100755 hsfs/python/hsfs/core/vector_server.py delete mode 100644 hsfs/python/hsfs/decorators.py delete mode 100644 hsfs/python/hsfs/embedding.py delete mode 100644 hsfs/python/hsfs/engine/__init__.py delete mode 100644 hsfs/python/hsfs/engine/python.py delete mode 100644 hsfs/python/hsfs/engine/spark.py delete mode 100644 hsfs/python/hsfs/engine/spark_no_metastore.py delete mode 100644 hsfs/python/hsfs/expectation_suite.py delete mode 100644 hsfs/python/hsfs/feature.py delete mode 100644 hsfs/python/hsfs/feature_group.py delete mode 100644 hsfs/python/hsfs/feature_group_commit.py delete mode 100644 hsfs/python/hsfs/feature_group_writer.py delete mode 100644 hsfs/python/hsfs/feature_store.py delete mode 100644 hsfs/python/hsfs/feature_view.py delete mode 100644 hsfs/python/hsfs/ge_expectation.py delete mode 100644 hsfs/python/hsfs/ge_validation_result.py delete mode 100644 hsfs/python/hsfs/hopsworks_udf.py delete mode 100644 hsfs/python/hsfs/serving_key.py delete mode 100644 hsfs/python/hsfs/split_statistics.py delete mode 100644 hsfs/python/hsfs/statistics.py delete mode 100644 hsfs/python/hsfs/statistics_config.py delete mode 100644 hsfs/python/hsfs/storage_connector.py delete mode 100644 hsfs/python/hsfs/tag.py delete mode 100644 hsfs/python/hsfs/training_dataset.py delete mode 100644 hsfs/python/hsfs/training_dataset_feature.py delete mode 100644 hsfs/python/hsfs/training_dataset_split.py delete mode 100644 hsfs/python/hsfs/transformation_function.py delete mode 100644 hsfs/python/hsfs/transformation_statistics.py delete mode 100644 hsfs/python/hsfs/usage.py delete mode 100644 hsfs/python/hsfs/user.py delete mode 100644 hsfs/python/hsfs/util.py delete mode 100644 hsfs/python/hsfs/validation_report.py delete mode 100644 hsfs/python/hsfs/version.py delete mode 100644 hsfs/python/pyproject.toml delete mode 100644 hsfs/python/setup.py delete mode 100644 hsfs/python/tests/__init__.py delete mode 100644 hsfs/python/tests/client/test_base_client.py delete mode 100644 hsfs/python/tests/conftest.py delete mode 100644 hsfs/python/tests/constructor/test_external_feature_group_alias.py delete mode 100644 hsfs/python/tests/constructor/test_filter.py delete mode 100644 hsfs/python/tests/constructor/test_fs_query.py delete mode 100644 hsfs/python/tests/constructor/test_hudi_feature_group_alias.py delete mode 100644 hsfs/python/tests/constructor/test_join.py delete mode 100644 hsfs/python/tests/constructor/test_prepared_statement_parameter.py delete mode 100644 hsfs/python/tests/constructor/test_query.py delete mode 100644 hsfs/python/tests/constructor/test_serving_prepared_statement.py delete mode 100644 hsfs/python/tests/core/__init__.py delete mode 100644 hsfs/python/tests/core/test_arrow_flight_client.py delete mode 100644 hsfs/python/tests/core/test_code_engine.py delete mode 100644 hsfs/python/tests/core/test_execution.py delete mode 100644 hsfs/python/tests/core/test_expectation_engine.py delete mode 100644 hsfs/python/tests/core/test_expectation_suite_engine.py delete mode 100644 hsfs/python/tests/core/test_external_feature_group_engine.py delete mode 100644 hsfs/python/tests/core/test_feature_descriptive_statistics.py delete mode 100644 hsfs/python/tests/core/test_feature_group_api.py delete mode 100644 hsfs/python/tests/core/test_feature_group_base_engine.py delete mode 100644 hsfs/python/tests/core/test_feature_group_engine.py delete mode 100644 hsfs/python/tests/core/test_feature_monitoring_config.py delete mode 100644 hsfs/python/tests/core/test_feature_monitoring_config_engine.py delete mode 100644 hsfs/python/tests/core/test_feature_monitoring_result.py delete mode 100644 hsfs/python/tests/core/test_feature_monitoring_result_engine.py delete mode 100644 hsfs/python/tests/core/test_feature_view_engine.py delete mode 100644 hsfs/python/tests/core/test_great_expectation_engine.py delete mode 100644 hsfs/python/tests/core/test_hudi_engine.py delete mode 100644 hsfs/python/tests/core/test_ingestion_job.py delete mode 100644 hsfs/python/tests/core/test_inode.py delete mode 100644 hsfs/python/tests/core/test_job.py delete mode 100644 hsfs/python/tests/core/test_job_configuration.py delete mode 100644 hsfs/python/tests/core/test_kafka_engine.py delete mode 100644 hsfs/python/tests/core/test_monitoring_window_config.py delete mode 100644 hsfs/python/tests/core/test_monitoring_window_config_engine.py delete mode 100644 hsfs/python/tests/core/test_online_store_rest_client.py delete mode 100644 hsfs/python/tests/core/test_online_store_rest_client_api.py delete mode 100644 hsfs/python/tests/core/test_online_store_rest_client_engine.py delete mode 100644 hsfs/python/tests/core/test_opensearch.py delete mode 100644 hsfs/python/tests/core/test_statistics_engine.py delete mode 100644 hsfs/python/tests/core/test_training_dataset_engine.py delete mode 100644 hsfs/python/tests/core/test_transformation_function_engine.py delete mode 100644 hsfs/python/tests/core/test_validation_report_engine.py delete mode 100644 hsfs/python/tests/core/test_validation_result_engine.py delete mode 100644 hsfs/python/tests/core/test_vector_db_client.py delete mode 100644 hsfs/python/tests/data/hadoop/bin/winutils.exe delete mode 100644 hsfs/python/tests/data/test_basic.csv delete mode 100644 hsfs/python/tests/data/test_basic.parquet delete mode 100644 hsfs/python/tests/data/test_basic.tsv delete mode 100644 hsfs/python/tests/engine/__init__.py delete mode 100644 hsfs/python/tests/engine/test_python.py delete mode 100644 hsfs/python/tests/engine/test_python_reader.py delete mode 100644 hsfs/python/tests/engine/test_python_spark_convert_dataframe.py delete mode 100644 hsfs/python/tests/engine/test_python_spark_transformation_functions.py delete mode 100644 hsfs/python/tests/engine/test_python_writer.py delete mode 100644 hsfs/python/tests/engine/test_spark.py delete mode 100644 hsfs/python/tests/fixtures/__init__.py delete mode 100644 hsfs/python/tests/fixtures/backend_fixtures.py delete mode 100644 hsfs/python/tests/fixtures/dataframe_fixtures.py delete mode 100644 hsfs/python/tests/fixtures/execution_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/expectation_suite_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/external_feature_group_alias_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/external_feature_group_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/feature_descriptive_statistics_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/feature_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/feature_group_commit_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/feature_group_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/feature_monitoring_config_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/feature_monitoring_result_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/feature_store_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/feature_view_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/filter_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/fs_query_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/ge_expectation_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/ge_validation_result_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/generate_backend_fixtures.ipynb delete mode 100644 hsfs/python/tests/fixtures/hudi_feature_group_alias_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/ingestion_job_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/inode_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/job_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/join_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/logic_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/prepared_statement_parameter_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/query_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/rondb_server_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/serving_keys_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/serving_prepared_statement_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/spine_group_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/split_statistics_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/statistics_config_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/statistics_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/storage_connector_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/tag_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/training_dataset_feature_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/training_dataset_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/training_dataset_split_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/transformation_function_attached_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/transformation_function_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/user_fixtures.json delete mode 100644 hsfs/python/tests/fixtures/validation_report_fixtures.json delete mode 100644 hsfs/python/tests/pyproject.toml delete mode 100644 hsfs/python/tests/test_expectation_suite.py delete mode 100644 hsfs/python/tests/test_feature.py delete mode 100644 hsfs/python/tests/test_feature_group.py delete mode 100644 hsfs/python/tests/test_feature_group_commit.py delete mode 100644 hsfs/python/tests/test_feature_group_writer.py delete mode 100644 hsfs/python/tests/test_feature_store.py delete mode 100644 hsfs/python/tests/test_feature_view.py delete mode 100644 hsfs/python/tests/test_ge_expectation.py delete mode 100644 hsfs/python/tests/test_ge_validation_result.py delete mode 100644 hsfs/python/tests/test_helpers/__init__.py delete mode 100644 hsfs/python/tests/test_helpers/transformation_test_helper.py delete mode 100644 hsfs/python/tests/test_hopswork_udf.py delete mode 100644 hsfs/python/tests/test_serving_keys.py delete mode 100644 hsfs/python/tests/test_split_statistics.py delete mode 100644 hsfs/python/tests/test_statistics.py delete mode 100644 hsfs/python/tests/test_statistics_config.py delete mode 100644 hsfs/python/tests/test_storage_connector.py delete mode 100644 hsfs/python/tests/test_tag.py delete mode 100644 hsfs/python/tests/test_training_dataset.py delete mode 100644 hsfs/python/tests/test_training_dataset_feature.py delete mode 100644 hsfs/python/tests/test_training_dataset_split.py delete mode 100644 hsfs/python/tests/test_transformation_function.py delete mode 100644 hsfs/python/tests/test_user.py delete mode 100644 hsfs/python/tests/test_util.py delete mode 100644 hsfs/python/tests/test_validation_report.py delete mode 100644 hsfs/python/tests/util.py delete mode 100644 hsfs/requirements-docs.txt delete mode 100644 hsfs/utils/java/pom.xml delete mode 100644 hsfs/utils/java/src/main/java/com/logicalclocks/utils/MainClass.java delete mode 100644 hsfs/utils/java/src/main/resources/checkstyle.xml delete mode 100644 hsfs/utils/java/src/main/resources/suppressions.xml delete mode 100644 hsfs/utils/python/hsfs_utils.py diff --git a/hsfs/.github/pull_request_template.md b/hsfs/.github/pull_request_template.md deleted file mode 100644 index 2a5a1e5ec..000000000 --- a/hsfs/.github/pull_request_template.md +++ /dev/null @@ -1,32 +0,0 @@ -This PR adds/fixes/changes... -- please summarize your changes to the code -- and make sure to include all changes to user-facing APIs - -JIRA Issue: - - -Priority for Review: - - -Related PRs: - - -**How Has This Been Tested?** - -- [ ] Unit Tests -- [ ] Integration Tests -- [ ] Manual Tests on VM - - -**Checklist For The Assigned Reviewer:** - -``` -- [ ] Checked if merge conflicts with master exist -- [ ] Checked if stylechecks for Java and Python pass -- [ ] Checked if all docstrings were added and/or updated appropriately -- [ ] Ran spellcheck on docstring -- [ ] Checked if guides & concepts need to be updated -- [ ] Checked if naming conventions for parameters and variables were followed -- [ ] Checked if private methods are properly declared and used -- [ ] Checked if hard-to-understand areas of code are commented -- [ ] Checked if tests are effective -- [ ] Built and deployed changes on dev VM and tested manually -- [x] (Checked if all type annotations were added and/or updated appropriately) -``` diff --git a/hsfs/.github/workflows/java-ut.yml b/hsfs/.github/workflows/java-ut.yml deleted file mode 100644 index f83f62caf..000000000 --- a/hsfs/.github/workflows/java-ut.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: java - -on: pull_request - -jobs: - unit_tests_utc: - name: Java Unit Tests - runs-on: ubuntu-latest - - steps: - - name: Set Timezone - run: sudo timedatectl set-timezone UTC - - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up JDK 8 - uses: actions/setup-java@v3 - with: - java-version: '8' - distribution: 'adopt' - - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Test - working-directory: ./java - run: mvn clean test - - unit_tests_local: - name: Java Unit Tests (Local TZ) - runs-on: ubuntu-latest - - steps: - - name: Set Timezone - run: sudo timedatectl set-timezone Europe/Amsterdam - - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up JDK 8 - uses: actions/setup-java@v3 - with: - java-version: '8' - distribution: 'adopt' - - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Test - working-directory: ./java - run: mvn clean test diff --git a/hsfs/.github/workflows/mkdocs-master.yml b/hsfs/.github/workflows/mkdocs-master.yml deleted file mode 100644 index 1c904ad28..000000000 --- a/hsfs/.github/workflows/mkdocs-master.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: mkdocs-master - -on: pull_request - -jobs: - publish-master: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: set dev version - working-directory: ./java - run: echo "DEV_VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -Ev 'Download|INFO|WARNING')" >> $GITHUB_ENV - - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: install deps - working-directory: ./python - run: cp ../README.md . && pip3 install -r ../requirements-docs.txt && pip3 install -e .[python,dev] - - - name: generate autodoc - run: python3 auto_doc.py - - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Set up JDK 8 - uses: actions/setup-java@v3 - with: - java-version: "8" - distribution: "adopt" - - - name: Build java doc documentation - working-directory: ./java - run: mvn clean install javadoc:javadoc javadoc:aggregate -DskipTests && cp -r target/site/apidocs ../docs/javadoc - - - name: setup git - run: | - git config --global user.name Mike - git config --global user.email mike@docs.hopsworks.ai - - - name: mike deploy docs - run: mike deploy ${{ env.DEV_VERSION }} dev -u diff --git a/hsfs/.github/workflows/mkdocs-release.yml b/hsfs/.github/workflows/mkdocs-release.yml deleted file mode 100644 index 66ca638ae..000000000 --- a/hsfs/.github/workflows/mkdocs-release.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: mkdocs-release - -on: - push: - branches: [branch-*] - -jobs: - publish-release: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: set major/minor/bugfix release version - working-directory: ./java - run: echo "RELEASE_VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -Ev 'Download|INFO|WARNING')" >> $GITHUB_ENV - - - name: set major/minor release version - run: echo "MAJOR_VERSION=$(echo $RELEASE_VERSION | sed 's/^\([0-9]*\.[0-9]*\).*$/\1/')" >> $GITHUB_ENV - - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: install deps - working-directory: ./python - run: cp ../README.md . && pip3 install -r ../requirements-docs.txt && pip3 install -e .[python,dev] - - - name: generate autodoc - run: python3 auto_doc.py - - - name: Cache local Maven repository - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('java/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - - name: Set up JDK 8 - uses: actions/setup-java@v3 - with: - java-version: "8" - distribution: "adopt" - - - name: Build java doc documentation - working-directory: ./java - run: mvn clean install javadoc:javadoc javadoc:aggregate -DskipTests && cp -r target/site/apidocs ../docs/javadoc - - - name: setup git - run: | - git config --global user.name Mike - git config --global user.email mike@docs.hopsworks.ai - - name: mike deploy docs - run: | - mike deploy ${{ env.RELEASE_VERSION }} ${{ env.MAJOR_VERSION }} -u --push - mike alias ${{ env.RELEASE_VERSION }} latest -u --push diff --git a/hsfs/.github/workflows/optional-dependency.yml b/hsfs/.github/workflows/optional-dependency.yml deleted file mode 100644 index 547b02029..000000000 --- a/hsfs/.github/workflows/optional-dependency.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: optional-dependency - -on: pull_request - -jobs: - unit_tests_no_great_expectations: - name: Unit Testing (No Great Expectations) - runs-on: ubuntu-latest - - steps: - - name: Set Timezone - run: sudo timedatectl set-timezone UTC - - - uses: actions/checkout@v4 - - name: Copy README - run: cp README.md python/ - - - uses: actions/setup-python@v5 - name: Setup Python - with: - python-version: "3.10" - cache: "pip" - cache-dependency-path: "python/setup.py" - - run: pip install -e python[python,dev-no-opt] - - - name: Run Pytest suite - env: - ENABLE_HOPSWORKS_USAGE: "false" - run: pytest python/tests \ No newline at end of file diff --git a/hsfs/.github/workflows/python-lint.yml b/hsfs/.github/workflows/python-lint.yml deleted file mode 100644 index f638b0128..000000000 --- a/hsfs/.github/workflows/python-lint.yml +++ /dev/null @@ -1,222 +0,0 @@ -name: python - -on: pull_request - -jobs: - lint_stylecheck: - name: Lint and Stylecheck - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Get all changed files - id: get-changed-files - uses: tj-actions/changed-files@v44 - with: - files_yaml: | - src: - - 'python/**/*.py' - - '!python/tests/**/*.py' - test: - - 'python/tests/**/*.py' - - - name: install deps - run: pip install ruff==0.4.2 - - - name: ruff on python files - if: steps.get-changed-files.outputs.src_any_changed == 'true' - env: - SRC_ALL_CHANGED_FILES: ${{ steps.get-changed-files.outputs.src_all_changed_files }} - run: ruff check --output-format=github $SRC_ALL_CHANGED_FILES - - - name: ruff on test files - if: steps.get-changed-files.outputs.test_any_changed == 'true' - env: - TEST_ALL_CHANGED_FILES: ${{ steps.get-changed-files.outputs.test_all_changed_files }} - run: ruff check --output-format=github $TEST_ALL_CHANGED_FILES - - - name: ruff format --check $ALL_CHANGED_FILES - env: - ALL_CHANGED_FILES: ${{ steps.get-changed-files.outputs.all_changed_files }} - run: ruff format $ALL_CHANGED_FILES - - unit_tests_ubuntu_utc: - name: Unit Testing (Ubuntu) - needs: lint_stylecheck - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10"] - - steps: - - name: Set Timezone - run: sudo timedatectl set-timezone UTC - - - uses: actions/checkout@v4 - - name: Copy README - run: cp README.md python/ - - - uses: actions/setup-python@v5 - name: Setup Python - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - cache-dependency-path: "python/setup.py" - - run: pip install -e python[python,dev] - - - name: Display Python version - run: python --version - - - name: Run Pytest suite - env: - ENABLE_HOPSWORKS_USAGE: "false" - run: pytest python/tests - - unit_tests_ubuntu_pandas: - name: Unit Testing (Ubuntu) (Pandas 1.x) - needs: lint_stylecheck - runs-on: ubuntu-latest - - steps: - - name: Set Timezone - run: sudo timedatectl set-timezone UTC - - - uses: actions/checkout@v4 - - name: Copy README - run: cp README.md python/ - - - uses: actions/setup-python@v4 - name: Setup Python - with: - python-version: "3.10" - cache: "pip" - cache-dependency-path: "python/setup.py" - - run: pip install -e python[python,dev-pandas1] - - - name: Display Python version - run: python --version - - - name: Run Pytest suite - env: - ENABLE_HOPSWORKS_USAGE: "false" - run: pytest python/tests - - unit_tests_ubuntu_local: - name: Unit Testing (Ubuntu) (Local TZ) - needs: lint_stylecheck - runs-on: ubuntu-latest - - steps: - - name: Set Timezone - run: sudo timedatectl set-timezone Europe/Amsterdam - - - uses: actions/checkout@v4 - - name: Copy README - run: cp README.md python/ - - - uses: actions/setup-python@v5 - name: Setup Python - with: - python-version: "3.10" - cache: "pip" - cache-dependency-path: "python/setup.py" - - run: pip install -e 'python[python,dev]' - - - name: Display Python version - run: python --version - - - name: Run Pytest suite - env: - ENABLE_HOPSWORKS_USAGE: "false" - run: pytest python/tests - - unit_tests_ubuntu_typechecked: - name: Typechecked Unit Testing (Ubuntu) - needs: lint_stylecheck - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Copy README - run: cp README.md python/ - - - uses: actions/setup-python@v4 - name: Setup Python - with: - python-version: "3.10" - cache: "pip" - cache-dependency-path: "python/setup.py" - - run: pip install -e python[python,dev,docs] - - - name: Run Pytest suite - env: - ENABLE_HOPSWORKS_USAGE: "false" - HOPSWORKS_RUN_WITH_TYPECHECK: "true" - run: pytest python/tests - continue-on-error: true - - unit_tests_windows_utc: - name: Unit Testing (Windows) - needs: lint_stylecheck - runs-on: windows-latest - - steps: - - name: Set Timezone - run: tzutil /s "UTC" - - - uses: actions/checkout@v4 - - name: Copy README - run: cp README.md python/ - - - uses: actions/setup-python@v5 - name: Setup Python - with: - python-version: "3.10" - cache: "pip" - cache-dependency-path: "python/setup.py" - - run: pip install -e python[python,dev] - - - name: Display Python version - run: python --version - - - name: Run Pytest suite - env: - ENABLE_HOPSWORKS_USAGE: "false" - run: pytest python/tests - - unit_tests_windows_local: - name: Unit Testing (Windows) (Local TZ) - needs: lint_stylecheck - runs-on: windows-latest - - steps: - - name: Set Timezone - run: tzutil /s "W. Europe Standard Time" - - - uses: actions/checkout@v4 - - name: Copy README - run: cp README.md python/ - - - uses: actions/setup-python@v5 - name: Setup Python - with: - python-version: "3.10" - cache: "pip" - cache-dependency-path: "python/setup.py" - - run: pip install -e python[python,dev] - - - name: Display Python version - run: python --version - - - name: Display pip freeze - run: pip freeze - - - name: Run Pytest suite - env: - ENABLE_HOPSWORKS_USAGE: "false" - run: pytest python/tests diff --git a/hsfs/.gitignore b/hsfs/.gitignore deleted file mode 100644 index a8b4c5683..000000000 --- a/hsfs/.gitignore +++ /dev/null @@ -1,145 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST -python/README.md -python/LICENSE - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -.pytest_cache/ -.ruff_cache/ -bigquery.json -metastore_db/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# Mike Javadoc -docs/javadoc - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# Java -.idea -.vscode -*.iml -target/ - -# Mac -.DS_Store - -# mkdocs intemediate files -docs/generated - -# Test artifacts -keyFile.json - -# delombok dir -delombok - -# dev scripts dir -dev_scripts/ -dev_tools/ diff --git a/hsfs/CONTRIBUTING.md b/hsfs/CONTRIBUTING.md deleted file mode 100644 index 0df3de08e..000000000 --- a/hsfs/CONTRIBUTING.md +++ /dev/null @@ -1,220 +0,0 @@ -## Python development setup - ---- - -- Fork and clone the repository - -- Create a new Python environment with your favourite environment manager (e.g. virtualenv or conda) and Python 3.9 (newer versions will return a library conflict in `auto_doc.py`) - -- Install repository in editable mode with development dependencies: - - ```bash - cd python - pip install -e ".[python,dev]" - ``` - -- Install [pre-commit](https://pre-commit.com/) and then activate its hooks. pre-commit is a framework for managing and maintaining multi-language pre-commit hooks. The Feature Store uses pre-commit to ensure code-style and code formatting through [ruff](https://docs.astral.sh/ruff/). Run the following commands from the `python` directory: - - ```bash - cd python - pip install --user pre-commit - pre-commit install - ``` - - Afterwards, pre-commit will run whenever you commit. - -- To run formatting and code-style separately, you can configure your IDE, such as VSCode, to use `ruff`, or run it via the command line: - - ```bash - # linting - ruff check python --fix - # formatting - ruff format python - ``` - -### Python documentation - -We follow a few best practices for writing the Python documentation: - -1. Use the google docstring style: - - ```python - """[One Line Summary] - - [Extended Summary] - - [!!! example - import xyz - ] - - # Arguments - arg1: Type[, optional]. Description[, defaults to `default`] - arg2: Type[, optional]. Description[, defaults to `default`] - - # Returns - Type. Description. - - # Raises - Exception. Description. - """ - ``` - - If Python 3 type annotations are used, they are inserted automatically. - -2. Feature store entity engine methods (e.g. FeatureGroupEngine etc.) only require a single line docstring. -3. REST Api implementations (e.g. FeatureGroupApi etc.) should be fully documented with docstrings without defaults. -4. Public Api such as metadata objects should be fully documented with defaults. - -#### Setup and Build Documentation - -We use `mkdocs` together with `mike` ([for versioning](https://github.com/jimporter/mike/)) to build the documentation and a plugin called `keras-autodoc` to auto generate Python API documentation from docstrings. - -**Background about `mike`:** -`mike` builds the documentation and commits it as a new directory to the gh-pages branch. Each directory corresponds to one version of the documentation. Additionally, `mike` maintains a json in the root of gh-pages with the mappings of versions/aliases for each of the directories available. With aliases you can define extra names like `dev` or `latest`, to indicate stable and unstable releases. - -1. Currently we are using our own version of `keras-autodoc` - - ```bash - pip install git+https://github.com/logicalclocks/keras-autodoc - ``` - -2. Install HSFS with `docs` extras: - - ```bash - pip install -e ".[python,dev]" && pip install -r ../requirements-docs.txt - ``` - -3. To build the docs, first run the auto doc script: - - ```bash - cd .. - python auto_doc.py - ``` - -##### Option 1: Build only current version of docs - -4. Either build the docs, or serve them dynamically: - - Note: Links and pictures might not resolve properly later on when checking with this build. - The reason for that is that the docs are deployed with versioning on docs.hopsworks.ai and - therefore another level is added to all paths, e.g. `docs.hopsworks.ai/[version-or-alias]`. - Using relative links should not be affected by this, however, building the docs with version - (Option 2) is recommended. - - ```bash - mkdocs build - # or - mkdocs serve - ``` - -##### Option 2 (Preferred): Build multi-version doc with `mike` - -###### Versioning on docs.hopsworks.ai - -On docs.hopsworks.ai we implement the following versioning scheme: - -- current master branches (e.g. of hsfs corresponding to master of Hopsworks): rendered as current Hopsworks snapshot version, e.g. **2.2.0-SNAPSHOT [dev]**, where `dev` is an alias to indicate that this is an unstable version. -- the latest release: rendered with full current version, e.g. **2.1.5 [latest]** with `latest` alias to indicate that this is the latest stable release. -- previous stable releases: rendered without alias, e.g. **2.1.4**. - -###### Build Instructions - -4. For this you can either checkout and make a local copy of the `upstream/gh-pages` branch, where `mike` maintains the current state of docs.hopsworks.ai, or just build documentation for the branch you are updating: - - Building _one_ branch: - - Checkout your dev branch with modified docs: - - ```bash - git checkout [dev-branch] - ``` - - Generate API docs if necessary: - - ```bash - python auto_doc.py - ``` - - Build docs with a version and alias - - ```bash - mike deploy [version] [alias] --update-alias - - # for example, if you are updating documentation to be merged to master, - # which will become the new SNAPSHOT version: - mike deploy 2.2.0-SNAPSHOT dev --update-alias - - # if you are updating docs of the latest stable release branch - mike deploy [version] latest --update-alias - - # if you are updating docs of a previous stable release branch - mike deploy [version] - ``` - - If no gh-pages branch existed in your local repository, this will have created it. - - **Important**: If no previous docs were built, you will have to choose a version as default to be loaded as index, as follows - - ```bash - mike set-default [version-or-alias] - ``` - - You can now checkout the gh-pages branch and serve: - - ```bash - git checkout gh-pages - mike serve - ``` - - You can also list all available versions/aliases: - - ```bash - mike list - ``` - - Delete and reset your local gh-pages branch: - - ```bash - mike delete --all - - # or delete single version - mike delete [version-or-alias] - ``` - -#### Adding new API documentation - -To add new documentation for APIs, you need to add information about the method/class to document to the `auto_doc.py` script: - -```python -PAGES = { - "connection.md": [ - "hsfs.connection.Connection.connection" - ] - "new_template.md": [ - "module", - "xyz.asd" - ] -} -``` - -Now you can add a template markdown file to the `docs/templates` directory with the name you specified in the auto-doc script. The `new_template.md` file should contain a tag to identify the place at which the API documentation should be inserted: - -```` -## The XYZ package - -{{module}} - -Some extra content here. - -!!! example - ```python - import xyz - ``` - -{{xyz.asd}} -```` - -Finally, run the `auto_doc.py` script, as decribed above, to update the documentation. - -For information about Markdown syntax and possible Admonitions/Highlighting etc. see -the [Material for Mkdocs themes reference documentation](https://squidfunk.github.io/mkdocs-material/reference/abbreviations/). diff --git a/hsfs/Dockerfile b/hsfs/Dockerfile deleted file mode 100644 index 38d9025c5..000000000 --- a/hsfs/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM ubuntu:22.04 - -RUN apt-get update && \ - apt-get install -y python3-pip git && apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN pip3 install twine build virtualenv \ - mkdocs==1.5.3 \ - mkdocs-material==9.5.17 \ - mike==2.0.0 \ - git+https://github.com/logicalclocks/keras-autodoc - -RUN mkdir -p /.local && chmod -R 777 /.local diff --git a/hsfs/Jenkinsfile b/hsfs/Jenkinsfile deleted file mode 100644 index d2014d5cb..000000000 --- a/hsfs/Jenkinsfile +++ /dev/null @@ -1,23 +0,0 @@ -pipeline { - agent { - docker { - label "local" - image "docker.hops.works/hopsworks_twine:0.0.1" - } - } - stages { - stage("publish") { - environment { - PYPI = credentials('977daeb0-e1c8-43a0-b35a-fc37bb9eee9b') - } - steps { - dir("python") { - sh "rm -f LICENSE README.md" - sh "cp -f ../LICENSE ../README.md ./" - sh "python3 -m build" - sh "twine upload -u $PYPI_USR -p $PYPI_PSW --skip-existing dist/*" - } - } - } - } -} diff --git a/hsfs/LICENSE b/hsfs/LICENSE deleted file mode 100644 index 261eeb9e9..000000000 --- a/hsfs/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/hsfs/README.md b/hsfs/README.md deleted file mode 100644 index a13ea2ce5..000000000 --- a/hsfs/README.md +++ /dev/null @@ -1,201 +0,0 @@ -# Hopsworks Feature Store - -

- Hopsworks Community - Hopsworks Feature Store Documentation - python - PyPiStatus - Scala/Java Artifacts - Downloads - Ruff - License -

- -HSFS is the library to interact with the Hopsworks Feature Store. The library makes creating new features, feature groups and training datasets easy. - -The library is environment independent and can be used in two modes: - -- Spark mode: For data engineering jobs that create and write features into the feature store or generate training datasets. It requires a Spark environment such as the one provided in the Hopsworks platform or Databricks. In Spark mode, HSFS provides bindings both for Python and JVM languages. - -- Python mode: For data science jobs to explore the features available in the feature store, generate training datasets and feed them in a training pipeline. Python mode requires just a Python interpreter and can be used both in Hopsworks from Python Jobs/Jupyter Kernels, Amazon SageMaker or KubeFlow. - -The library automatically configures itself based on the environment it is run. -However, to connect from an external environment such as Databricks or AWS Sagemaker, -additional connection information, such as host and port, is required. For more information checkout the [Hopsworks documentation](https://docs.hopsworks.ai/latest/). - -## Getting Started On Hopsworks - -Get started easily by registering an account on [Hopsworks Serverless](https://app.hopsworks.ai/). Create your project and a [new Api key](https://docs.hopsworks.ai/latest/user_guides/projects/api_key/create_api_key/). In a new python environment with Python 3.8 or higher, install the [client library](https://docs.hopsworks.ai/latest/user_guides/client_installation/) using pip: - -```bash -# Get all Hopsworks SDKs: Feature Store, Model Serving and Platform SDK -pip install hopsworks -# or minimum install with the Feature Store SDK -pip install hsfs[python] -# if using zsh don't forget the quotes -pip install 'hsfs[python]' -``` - -You can start a notebook and instantiate a connection and get the project feature store handler. - -```python -import hopsworks - -project = hopsworks.login() # you will be prompted for your api key -fs = project.get_feature_store() -``` - -or using `hsfs` directly: - -```python -import hsfs - -connection = hsfs.connection( - host="c.app.hopsworks.ai", # - project="your-project", - api_key_value="your-api-key", -) -fs = connection.get_feature_store() -``` - -Create a new feature group to start inserting feature values. -```python -fg = fs.create_feature_group("rain", - version=1, - description="Rain features", - primary_key=['date', 'location_id'], - online_enabled=True) - -fg.save(dataframe) -``` - -Upsert new data in to the feature group with `time_travel_format="HUDI"`". -```python -fg.insert(upsert_df) -``` - -Retrieve commit timeline metdata of the feature group with `time_travel_format="HUDI"`". -```python -fg.commit_details() -``` - -"Reading feature group as of specific point in time". -```python -fg = fs.get_feature_group("rain", 1) -fg.read("2020-10-20 07:34:11").show() -``` - -Read updates that occurred between specified points in time. -```python -fg = fs.get_feature_group("rain", 1) -fg.read_changes("2020-10-20 07:31:38", "2020-10-20 07:34:11").show() -``` - -Join features together -```python -feature_join = rain_fg.select_all() - .join(temperature_fg.select_all(), on=["date", "location_id"]) - .join(location_fg.select_all()) -feature_join.show(5) -``` - -join feature groups that correspond to specific point in time -```python -feature_join = rain_fg.select_all() - .join(temperature_fg.select_all(), on=["date", "location_id"]) - .join(location_fg.select_all()) - .as_of("2020-10-31") -feature_join.show(5) -``` - -join feature groups that correspond to different time -```python -rain_fg_q = rain_fg.select_all().as_of("2020-10-20 07:41:43") -temperature_fg_q = temperature_fg.select_all().as_of("2020-10-20 07:32:33") -location_fg_q = location_fg.select_all().as_of("2020-10-20 07:33:08") -joined_features_q = rain_fg_q.join(temperature_fg_q).join(location_fg_q) -``` - -Use the query object to create a training dataset: -```python -td = fs.create_training_dataset("rain_dataset", - version=1, - data_format="tfrecords", - description="A test training dataset saved in TfRecords format", - splits={'train': 0.7, 'test': 0.2, 'validate': 0.1}) - -td.save(feature_join) -``` - -A short introduction to the Scala API: -```scala -import com.logicalclocks.hsfs._ -val connection = HopsworksConnection.builder().build() -val fs = connection.getFeatureStore(); -val attendances_features_fg = fs.getFeatureGroup("games_features", 1); -attendances_features_fg.show(1) -``` - -You can find more examples on how to use the library in our [hops-examples](https://github.com/logicalclocks/hops-examples) repository. - -## Usage - -Usage data is collected for improving quality of the library. It is turned on by default if the backend -is "c.app.hopsworks.ai". To turn it off, use one of the following way: -```python -# use environment variable -import os -os.environ["ENABLE_HOPSWORKS_USAGE"] = "false" - -# use `disable_usage_logging` -import hsfs -hsfs.disable_usage_logging() -``` - -The source code can be found in python/hsfs/usage.py. - -## Documentation - -Documentation is available at [Hopsworks Feature Store Documentation](https://docs.hopsworks.ai/). - -## Issues - -For general questions about the usage of Hopsworks and the Feature Store please open a topic on [Hopsworks Community](https://community.hopsworks.ai/). - -Please report any issue using [Github issue tracking](https://github.com/logicalclocks/feature-store-api/issues). - -Please attach the client environment from the output below in the issue: -```python -import hopsworks -import hsfs -hopsworks.login().get_feature_store() -print(hsfs.get_env()) -``` - -## Contributing - -If you would like to contribute to this library, please see the [Contribution Guidelines](CONTRIBUTING.md). diff --git a/hsfs/auto_doc.py b/hsfs/auto_doc.py deleted file mode 100644 index a98af258b..000000000 --- a/hsfs/auto_doc.py +++ /dev/null @@ -1,384 +0,0 @@ -import os -import pathlib -import shutil - -import keras_autodoc - -PAGES = { - "api/connection_api.md": { - "connection": ["hsfs.connection.Connection"], - "connection_properties": keras_autodoc.get_properties( - "hsfs.connection.Connection" - ), - "connection_methods": keras_autodoc.get_methods("hsfs.connection.Connection"), - }, - "api/spine_group_api.md": { - "fg": ["hsfs.feature_group.SpineGroup"], - "fg_create": ["hsfs.feature_store.FeatureStore.get_or_create_spine_group"], - "fg_get": ["hsfs.feature_store.FeatureStore.get_or_create_spine_group"], - "fg_properties": keras_autodoc.get_properties( - "hsfs.feature_group.SpineGroup", - exclude=[ - "expectation_suite", - "location", - "online_enabled", - "statistics", - "statistics_config", - "subject", - ], - ), - "fg_methods": keras_autodoc.get_methods( - "hsfs.feature_group.SpineGroup", - exclude=[ - "append_features", - "compute_statistics", - "delete_expectation_suite", - "from_response_json", - "get_all_validation_reports", - "get_expectation_suite", - "get_latest_validation_report", - "get_statistics", - "get_validation_history", - "save_expectation_suite", - "save_validation_report", - "update_from_response_json", - "update_statistics_config", - "validate", - ], - ), - }, - "api/training_dataset_api.md": { - "td": ["hsfs.training_dataset.TrainingDataset"], - "td_create": ["hsfs.feature_store.FeatureStore.create_training_dataset"], - "td_get": ["hsfs.feature_store.FeatureStore.get_training_dataset"], - "td_properties": keras_autodoc.get_properties( - "hsfs.training_dataset.TrainingDataset" - ), - "td_methods": keras_autodoc.get_methods( - "hsfs.training_dataset.TrainingDataset" - ), - }, - "api/feature_view_api.md": { - "fv": ["hsfs.feature_view.FeatureView"], - "fv_create": ["hsfs.feature_store.FeatureStore.create_feature_view"], - "fv_get": ["hsfs.feature_store.FeatureStore.get_feature_view"], - "fvs_get": ["hsfs.feature_store.FeatureStore.get_feature_views"], - "fv_properties": keras_autodoc.get_properties("hsfs.feature_view.FeatureView"), - "fv_methods": keras_autodoc.get_methods("hsfs.feature_view.FeatureView"), - }, - "api/feature_api.md": { - "feature": ["hsfs.feature.Feature"], - "feature_properties": keras_autodoc.get_properties("hsfs.feature.Feature"), - "feature_methods": keras_autodoc.get_methods("hsfs.feature.Feature"), - }, - "api/expectation_suite_api.md": { - "expectation_suite": ["hsfs.expectation_suite.ExpectationSuite"], - "expectation_suite_attach": [ - "hsfs.feature_group.FeatureGroup.save_expectation_suite" - ], - "single_expectation_api": [ - "hsfs.expectation_suite.ExpectationSuite.add_expectation", - "hsfs.expectation_suite.ExpectationSuite.replace_expectation", - "hsfs.expectation_suite.ExpectationSuite.remove_expectation", - ], - "expectation_suite_properties": keras_autodoc.get_properties( - "hsfs.expectation_suite.ExpectationSuite" - ), - "expectation_suite_methods": keras_autodoc.get_methods( - "hsfs.expectation_suite.ExpectationSuite" - ), - }, - "api/feature_store_api.md": { - "fs": ["hsfs.feature_store.FeatureStore"], - "fs_get": ["hsfs.connection.Connection.get_feature_store"], - "fs_properties": keras_autodoc.get_properties( - "hsfs.feature_store.FeatureStore" - ), - "fs_methods": keras_autodoc.get_methods("hsfs.feature_store.FeatureStore"), - }, - "api/feature_group_api.md": { - "fg": ["hsfs.feature_group.FeatureGroup"], - "fg_create": [ - "hsfs.feature_store.FeatureStore.create_feature_group", - "hsfs.feature_store.FeatureStore.get_or_create_feature_group", - ], - "fg_get": ["hsfs.feature_store.FeatureStore.get_feature_group"], - "fg_properties": keras_autodoc.get_properties( - "hsfs.feature_group.FeatureGroup" - ), - "fg_methods": keras_autodoc.get_methods("hsfs.feature_group.FeatureGroup"), - }, - "api/external_feature_group_api.md": { - "fg": ["hsfs.feature_group.ExternalFeatureGroup"], - "fg_create": ["hsfs.feature_store.FeatureStore.create_external_feature_group"], - "fg_get": ["hsfs.feature_store.FeatureStore.get_external_feature_group"], - "fg_properties": keras_autodoc.get_properties( - "hsfs.feature_group.ExternalFeatureGroup" - ), - "fg_methods": keras_autodoc.get_methods( - "hsfs.feature_group.ExternalFeatureGroup" - ), - }, - "api/storage_connector_api.md": { - "sc_get": [ - "hsfs.feature_store.FeatureStore.get_storage_connector", - "hsfs.feature_store.FeatureStore.get_online_storage_connector", - ], - "hopsfs_methods": keras_autodoc.get_methods( - "hsfs.storage_connector.HopsFSConnector", exclude=["from_response_json"] - ), - "hopsfs_properties": keras_autodoc.get_properties( - "hsfs.storage_connector.HopsFSConnector" - ), - "s3_methods": keras_autodoc.get_methods( - "hsfs.storage_connector.S3Connector", exclude=["from_response_json"] - ), - "s3_properties": keras_autodoc.get_properties( - "hsfs.storage_connector.S3Connector" - ), - "redshift_methods": keras_autodoc.get_methods( - "hsfs.storage_connector.RedshiftConnector", exclude=["from_response_json"] - ), - "redshift_properties": keras_autodoc.get_properties( - "hsfs.storage_connector.RedshiftConnector" - ), - "adls_methods": keras_autodoc.get_methods( - "hsfs.storage_connector.AdlsConnector", exclude=["from_response_json"] - ), - "adls_properties": keras_autodoc.get_properties( - "hsfs.storage_connector.AdlsConnector" - ), - "snowflake_methods": keras_autodoc.get_methods( - "hsfs.storage_connector.SnowflakeConnector", exclude=["from_response_json"] - ), - "snowflake_properties": keras_autodoc.get_properties( - "hsfs.storage_connector.SnowflakeConnector" - ), - "jdbc_methods": keras_autodoc.get_methods( - "hsfs.storage_connector.JdbcConnector", exclude=["from_response_json"] - ), - "jdbc_properties": keras_autodoc.get_properties( - "hsfs.storage_connector.JdbcConnector" - ), - "gcs_methods": keras_autodoc.get_methods( - "hsfs.storage_connector.GcsConnector", exclude=["from_response_json"] - ), - "gcs_properties": keras_autodoc.get_properties( - "hsfs.storage_connector.GcsConnector" - ), - "bigquery_methods": keras_autodoc.get_methods( - "hsfs.storage_connector.BigQueryConnector", exclude=["from_response_json"] - ), - "bigquery_properties": keras_autodoc.get_properties( - "hsfs.storage_connector.BigQueryConnector" - ), - "kafka_methods": keras_autodoc.get_methods( - "hsfs.storage_connector.KafkaConnector", exclude=["from_response_json"] - ), - "kafka_properties": keras_autodoc.get_properties( - "hsfs.storage_connector.KafkaConnector" - ), - }, - "api/statistics_config_api.md": { - "statistics_config": ["hsfs.statistics_config.StatisticsConfig"], - "statistics_config_properties": keras_autodoc.get_properties( - "hsfs.statistics_config.StatisticsConfig" - ), - }, - "api/transformation_functions_api.md": { - "transformation_function": [ - "hsfs.transformation_function.TransformationFunction" - ], - "transformation_function_properties": keras_autodoc.get_properties( - "hsfs.transformation_function.TransformationFunction" - ), - "transformation_function_methods": keras_autodoc.get_methods( - "hsfs.transformation_function.TransformationFunction", - exclude=[ - "from_response_json", - "update_from_response_json", - "json", - "to_dict", - ], - ), - "create_transformation_function": [ - "hsfs.feature_store.FeatureStore.create_transformation_function" - ], - "get_transformation_function": [ - "hsfs.feature_store.FeatureStore.get_transformation_function" - ], - "get_transformation_functions": [ - "hsfs.feature_store.FeatureStore.get_transformation_functions" - ], - }, - "api/validation_report_api.md": { - "validation_report": ["hsfs.validation_report.ValidationReport"], - "validation_report_validate": [ - "hsfs.feature_group.FeatureGroup.validate", - "hsfs.feature_group.FeatureGroup.insert", - ], - "validation_report_get": [ - "hsfs.feature_group.FeatureGroup.get_latest_validation_report", - "hsfs.feature_group.FeatureGroup.get_all_validation_reports", - ], - "validation_report_properties": keras_autodoc.get_properties( - "hsfs.validation_report.ValidationReport" - ), - "validation_report_methods": keras_autodoc.get_methods( - "hsfs.validation_report.ValidationReport" - ), - }, - "api/job.md": { - "job_configuration": ["hsfs.core.job_configuration.JobConfiguration"], - "job": ["hsfs.core.job.Job"], - "job_methods": [ - "hsfs.core.job.Job.get_state", - "hsfs.core.job.Job.get_final_state", - ], - }, - "api/query_api.md": { - "query_methods": keras_autodoc.get_methods( - "hsfs.constructor.query.Query", - exclude=["json", "to_dict"], - ), - "query_properties": keras_autodoc.get_properties( - "hsfs.constructor.query.Query" - ), - }, - "api/links.md": { - "links_properties": keras_autodoc.get_properties( - "hsfs.core.explicit_provenance.Links" - ), - "artifact_properties": keras_autodoc.get_properties( - "hsfs.core.explicit_provenance.Artifact" - ), - }, - "api/statistics_api.md": { - "statistics": ["hsfs.statistics.Statistics"], - "statistics_properties": keras_autodoc.get_properties( - "hsfs.statistics.Statistics" - ), - }, - "api/split_statistics_api.md": { - "split_statistics": ["hsfs.split_statistics.SplitStatistics"], - "split_statistics_properties": keras_autodoc.get_properties( - "hsfs.split_statistics.SplitStatistics" - ), - }, - "api/feature_descriptive_statistics_api.md": { - "feature_descriptive_statistics": [ - "hsfs.core.feature_descriptive_statistics.FeatureDescriptiveStatistics" - ], - "feature_descriptive_statistics_properties": keras_autodoc.get_properties( - "hsfs.core.feature_descriptive_statistics.FeatureDescriptiveStatistics" - ), - }, - "api/feature_monitoring_config_api.md": { - "feature_monitoring_config": [ - "hsfs.core.feature_monitoring_config.FeatureMonitoringConfig" - ], - "feature_monitoring_config_properties": keras_autodoc.get_properties( - "hsfs.core.feature_monitoring_config.FeatureMonitoringConfig" - ), - "feature_monitoring_config_methods": keras_autodoc.get_methods( - "hsfs.core.feature_monitoring_config.FeatureMonitoringConfig", - exclude=[ - "from_response_json", - "update_from_response_json", - "json", - "to_dict", - ], - ), - # from feature group - "feature_monitoring_config_creation_fg": [ - "hsfs.feature_group.FeatureGroup.create_statistics_monitoring", - "hsfs.feature_group.FeatureGroup.create_feature_monitoring", - ], - # from feature view - "feature_monitoring_config_creation_fv": [ - "hsfs.feature_view.FeatureView.create_statistics_monitoring", - "hsfs.feature_view.FeatureView.create_feature_monitoring", - ], - # retrieval - "feature_monitoring_config_retrieval_fg": [ - "hsfs.feature_group.FeatureGroup.get_feature_monitoring_configs", - ], - "feature_monitoring_config_retrieval_fv": [ - "hsfs.feature_view.FeatureView.get_feature_monitoring_configs", - ], - }, - "api/feature_monitoring_result_api.md": { - "feature_monitoring_result": [ - "hsfs.core.feature_monitoring_result.FeatureMonitoringResult" - ], - "feature_monitoring_result_retrieval": [ - "hsfs.core.feature_monitoring_config.FeatureMonitoringConfig.get_history" - ], - "feature_monitoring_result_properties": keras_autodoc.get_properties( - "hsfs.core.feature_monitoring_result.FeatureMonitoringResult" - ), - }, - "api/feature_monitoring_window_config_api.md": { - "feature_monitoring_window_config": [ - "hsfs.core.monitoring_window_config.MonitoringWindowConfig" - ], - "feature_monitoring_window_config_properties": keras_autodoc.get_properties( - "hsfs.core.monitoring_window_config.MonitoringWindowConfig" - ), - }, - "api/embedding_index_api.md": { - "embedding_index": ["hsfs.embedding.EmbeddingIndex"], - "embedding_index_properties": keras_autodoc.get_properties( - "hsfs.embedding.EmbeddingIndex" - ), - "embedding_index_methods": keras_autodoc.get_methods( - "hsfs.embedding.EmbeddingIndex", exclude=["from_response_json"] - ), - }, - "api/embedding_feature_api.md": { - "embedding_feature": ["hsfs.embedding.EmbeddingFeature"], - "embedding_feature_properties": keras_autodoc.get_properties( - "hsfs.embedding.EmbeddingFeature" - ), - }, - "api/similarity_function_type_api.md": { - "similarity_function_type": ["hsfs.embedding.SimilarityFunctionType"], - }, -} - -hsfs_dir = pathlib.Path(__file__).resolve().parents[0] -if "GITHUB_SHA" in os.environ: - commit_sha = os.environ["GITHUB_SHA"] - project_url = ( - f"https://github.com/logicalclocks/feature-store-api/tree/{commit_sha}/python" - ) -else: - branch_name = os.environ.get("GITHUB_BASE_REF", "master") - project_url = ( - f"https://github.com/logicalclocks/feature-store-api/blob/{branch_name}/python" - ) - - -def generate(dest_dir): - doc_generator = keras_autodoc.DocumentationGenerator( - PAGES, - project_url=project_url, - template_dir="./docs/templates", - titles_size="###", - extra_aliases={ - "hsfs.core.query.Query": "hsfs.Query", - "hsfs.storage_connector.StorageConnector": "hsfs.StorageConnector", - "hsfs.statistics_config.StatisticsConfig": "hsfs.StatisticsConfig", - "hsfs.training_dataset_feature.TrainingDatasetFeature": "hsfs.TrainingDatasetFeature", - "pandas.core.frame.DataFrame": "pandas.DataFrame", - }, - max_signature_line_length=100, - ) - shutil.copyfile(hsfs_dir / "CONTRIBUTING.md", dest_dir / "CONTRIBUTING.md") - shutil.copyfile(hsfs_dir / "README.md", dest_dir / "index.md") - - doc_generator.generate(dest_dir / "generated") - - -if __name__ == "__main__": - generate(hsfs_dir / "docs") diff --git a/hsfs/docs/CONTRIBUTING.md b/hsfs/docs/CONTRIBUTING.md deleted file mode 100644 index 0df3de08e..000000000 --- a/hsfs/docs/CONTRIBUTING.md +++ /dev/null @@ -1,220 +0,0 @@ -## Python development setup - ---- - -- Fork and clone the repository - -- Create a new Python environment with your favourite environment manager (e.g. virtualenv or conda) and Python 3.9 (newer versions will return a library conflict in `auto_doc.py`) - -- Install repository in editable mode with development dependencies: - - ```bash - cd python - pip install -e ".[python,dev]" - ``` - -- Install [pre-commit](https://pre-commit.com/) and then activate its hooks. pre-commit is a framework for managing and maintaining multi-language pre-commit hooks. The Feature Store uses pre-commit to ensure code-style and code formatting through [ruff](https://docs.astral.sh/ruff/). Run the following commands from the `python` directory: - - ```bash - cd python - pip install --user pre-commit - pre-commit install - ``` - - Afterwards, pre-commit will run whenever you commit. - -- To run formatting and code-style separately, you can configure your IDE, such as VSCode, to use `ruff`, or run it via the command line: - - ```bash - # linting - ruff check python --fix - # formatting - ruff format python - ``` - -### Python documentation - -We follow a few best practices for writing the Python documentation: - -1. Use the google docstring style: - - ```python - """[One Line Summary] - - [Extended Summary] - - [!!! example - import xyz - ] - - # Arguments - arg1: Type[, optional]. Description[, defaults to `default`] - arg2: Type[, optional]. Description[, defaults to `default`] - - # Returns - Type. Description. - - # Raises - Exception. Description. - """ - ``` - - If Python 3 type annotations are used, they are inserted automatically. - -2. Feature store entity engine methods (e.g. FeatureGroupEngine etc.) only require a single line docstring. -3. REST Api implementations (e.g. FeatureGroupApi etc.) should be fully documented with docstrings without defaults. -4. Public Api such as metadata objects should be fully documented with defaults. - -#### Setup and Build Documentation - -We use `mkdocs` together with `mike` ([for versioning](https://github.com/jimporter/mike/)) to build the documentation and a plugin called `keras-autodoc` to auto generate Python API documentation from docstrings. - -**Background about `mike`:** -`mike` builds the documentation and commits it as a new directory to the gh-pages branch. Each directory corresponds to one version of the documentation. Additionally, `mike` maintains a json in the root of gh-pages with the mappings of versions/aliases for each of the directories available. With aliases you can define extra names like `dev` or `latest`, to indicate stable and unstable releases. - -1. Currently we are using our own version of `keras-autodoc` - - ```bash - pip install git+https://github.com/logicalclocks/keras-autodoc - ``` - -2. Install HSFS with `docs` extras: - - ```bash - pip install -e ".[python,dev]" && pip install -r ../requirements-docs.txt - ``` - -3. To build the docs, first run the auto doc script: - - ```bash - cd .. - python auto_doc.py - ``` - -##### Option 1: Build only current version of docs - -4. Either build the docs, or serve them dynamically: - - Note: Links and pictures might not resolve properly later on when checking with this build. - The reason for that is that the docs are deployed with versioning on docs.hopsworks.ai and - therefore another level is added to all paths, e.g. `docs.hopsworks.ai/[version-or-alias]`. - Using relative links should not be affected by this, however, building the docs with version - (Option 2) is recommended. - - ```bash - mkdocs build - # or - mkdocs serve - ``` - -##### Option 2 (Preferred): Build multi-version doc with `mike` - -###### Versioning on docs.hopsworks.ai - -On docs.hopsworks.ai we implement the following versioning scheme: - -- current master branches (e.g. of hsfs corresponding to master of Hopsworks): rendered as current Hopsworks snapshot version, e.g. **2.2.0-SNAPSHOT [dev]**, where `dev` is an alias to indicate that this is an unstable version. -- the latest release: rendered with full current version, e.g. **2.1.5 [latest]** with `latest` alias to indicate that this is the latest stable release. -- previous stable releases: rendered without alias, e.g. **2.1.4**. - -###### Build Instructions - -4. For this you can either checkout and make a local copy of the `upstream/gh-pages` branch, where `mike` maintains the current state of docs.hopsworks.ai, or just build documentation for the branch you are updating: - - Building _one_ branch: - - Checkout your dev branch with modified docs: - - ```bash - git checkout [dev-branch] - ``` - - Generate API docs if necessary: - - ```bash - python auto_doc.py - ``` - - Build docs with a version and alias - - ```bash - mike deploy [version] [alias] --update-alias - - # for example, if you are updating documentation to be merged to master, - # which will become the new SNAPSHOT version: - mike deploy 2.2.0-SNAPSHOT dev --update-alias - - # if you are updating docs of the latest stable release branch - mike deploy [version] latest --update-alias - - # if you are updating docs of a previous stable release branch - mike deploy [version] - ``` - - If no gh-pages branch existed in your local repository, this will have created it. - - **Important**: If no previous docs were built, you will have to choose a version as default to be loaded as index, as follows - - ```bash - mike set-default [version-or-alias] - ``` - - You can now checkout the gh-pages branch and serve: - - ```bash - git checkout gh-pages - mike serve - ``` - - You can also list all available versions/aliases: - - ```bash - mike list - ``` - - Delete and reset your local gh-pages branch: - - ```bash - mike delete --all - - # or delete single version - mike delete [version-or-alias] - ``` - -#### Adding new API documentation - -To add new documentation for APIs, you need to add information about the method/class to document to the `auto_doc.py` script: - -```python -PAGES = { - "connection.md": [ - "hsfs.connection.Connection.connection" - ] - "new_template.md": [ - "module", - "xyz.asd" - ] -} -``` - -Now you can add a template markdown file to the `docs/templates` directory with the name you specified in the auto-doc script. The `new_template.md` file should contain a tag to identify the place at which the API documentation should be inserted: - -```` -## The XYZ package - -{{module}} - -Some extra content here. - -!!! example - ```python - import xyz - ``` - -{{xyz.asd}} -```` - -Finally, run the `auto_doc.py` script, as decribed above, to update the documentation. - -For information about Markdown syntax and possible Admonitions/Highlighting etc. see -the [Material for Mkdocs themes reference documentation](https://squidfunk.github.io/mkdocs-material/reference/abbreviations/). diff --git a/hsfs/docs/assets/images/favicon.ico b/hsfs/docs/assets/images/favicon.ico deleted file mode 100644 index ab757306798d8da0cea9ca008ac05cd5091eff1a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2699 zcmb_e3se(V8jfIHc?nn*RJ79pw1AV$Ofq>)i3)^Atl?#-u~uX>Y4kGiR{>X1m?N-aL#6>&zU^vKO4!v?J2J z$2goBEn20@qEf8JNk-2lC`L_lZF(b%=5Qic*o*|3O%YjH6&JyTp3jx^lI)fRvNx?B*oNYV1c_1(bv1CiZ zC}%)GrAP*17!wT$xiCb+FbaqyTtq0~W9Tvfg;AIXBRoC~AsCLqIA07*{6IFEiPGSy zvbc#@?2{DCvRI5bk7u=7xmG@xF=g@)iA2JKQ67pyECMp;87zbiGMIxW8DzAXG--_% zEn@(jjD(uWu}DEy>39lyql?yHo+uMrFrJMt@(?cUOlb^AkuIDu$D|tzPLVuXN9$>W z#mr(67uJ}?SQv8_^8{+w@RR^-X%z}rjK^xx>s=wtme^c2jfsLh7HwAM8EIZBZDw*z zBpsW}$_#d9W5i=jG+|*(N`}!*E>!ZQWB_F|0iIi{HBgMz9QMcrT1HrCDd=1^2o^xF zM2QG+fdofH5ip9w@HkY#P+CpiR49r%p@;~_rb1cEP=tl}RWL>38pfn2*o3uuB9rDB z4VfU|DkC1l=ok|lm{rGrv|k<*lWbx%S{*xJPK}QSI%%t5D!_gV{nD zNLntFf^2cPS}lcR5)n#kNEG73s1Op70vHlY#3D$9i3p(@Lr4iBnw&3V$Q-8yob$(b z1I3Um$ItQ*kqA@sDLy11UX ziV=K5OhXu~Mj;pzNFcFDOh7QDK_mhpfg*^|Y5OrJ;R#wZyT*AF8)*&w%0x@21;)&S z6QpxfNI}x+2ATpVy0wp~$8U-}G2fa+v!uV^lJV$fMq{xOCORsUt=+G>koP3wPlp?c$ZXK4M#t4HUqZe105YgPfj z-g{}u{L|Yy2Uj;vw+}AqYJE=l?lZop64Zeeb3u46M3A`aJ%t>ujl<`6oIMp+yt^nUptAEm zh^#o7^<6+s+Lew2p+Rpxy=ALs%h`;<#=-q(ebRbAqy~;68oj)#@1-yH^(Pl}ua}Md z310Wsj7ueUnJ#|2&rt1ajpnNS{W^c*azc9bKS%db0m@TfDMCJ3`RO~_j~CwnhNds? zNXz-jvg5>$Gm^TcDf*mVG$rs}ud#pL>?Y9XV0y>HxVyt$l|HW@4D2XgkVTv@!lQ8?>o`iF4 z(TBh7al=}kM%}DEr3(s|`}f(4zi4ug{n!4ws;aL)HnfFFT0aXovUxV@_WnKZ{#C7S zc5km*-ne@M-Jt28^Wnp|PgS9B{3GvX-3L^_JxM~yKtP|J*)+V!w*=p{uu4^6xLbL& z{UEjQRS)`(XJK?pbsNv8{onB!NsHXpm5-j@SrL0Vn*Uz&rlRBv)#@LYgx|83`2KPA zM`zwHQ$)w!yfIV%m9{QJ`%BF?61%-e-LjvmZ?$UCTIc7Y NJT_5wGCE`1e*yWD-b4TZ diff --git a/hsfs/docs/assets/images/hops-logo.png b/hsfs/docs/assets/images/hops-logo.png deleted file mode 100644 index d3625ae07da68430e3bd5f46a1a7baf88dd474cc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6356 zcmb`M*H;tH^Y;T%LX#x)4nb-Vq>3~l2BZ@ZFcd*UmEHtJlom=tl}-dC6e-e+5S1pO zgY-~Dsx+m86oHQq-@o9w`CZJ}b7tQ&XJ>A9=e1a*@qH$GZh8O!z@(>(FarQ6gsw~i zNOR?xGfoSmU+Q-MS^0Er4r8CEI?2Y}cRKtMqRkV4ZOBm{`1hf)DdwJ-od z$yg8oLH&OwTD^b$?(-r~yfgaC<;1~5bw#84)X>*IG3nG$Du--t^f*^_Q(n@RHD=Qk z;r4IVN9i!~(UFgIU_8nb5%6}Wr-B&)6osnPQ~iM1A4PM5Hxe14=3%>A*y8G`T^vY8XIGgU{= zlMnB86hh`|)w1-1PQ8K1x{x%^(Td}11ZuGUTgzn)`=`~t5e$6=KUiQY0&}$i9e?`Y z+EqZ&+SOqqY5cji&CPZ%{Ds)JjJ9c)6+wgsl!Pjs6XRpp~M;Ro==k+rp zO0GlwC503rU{TLXDe~pVic3x22$8hwm1-h=5$Vi|+B7KmW=(2vzQt8ArAGg?lHlJ> zT9_0EP{HR}W(2jZetWRldAKHK%+b62pY6F;Y1pfr4byKyjUX2Nw^-Yv35N9H;YMR0 z5=8$kq1Exxd3b5qg4<0G1ZK=djD0loe@QOP3KcQH)lRtKaqIClQ{{(^bYNM+MOLr2 zzPjnm+$5x?`>I)DW2Y9#1nnzf-#6WrmArHVHaJE#<8ChF=vQ@EX2BzQg!uJbU(;>g zV&_c8HW@rR3J9h*fT4Yr>sE~G>>YTx#n5cE{;h^MRwzQ8*8q&qEPpoz@!^LdWa#+P zw0z+h;F*hLN0)y^>il$tEc?)I)E9C6*xtmmLbj=m4{+eI<(L3kHeuCtuKxF6BF~p^ z7J8kdHJFQ<5ASI-_CPybD zYIDXbU0GncY*K=g6!O$q7uAVP&YK8sXzg|Jdm0^dkoWNV0?XHZp%SC&*yP0T2ya6H zOH+r~QN{srSRK8&Fnm3=9FToFyM=w5ApoN@7h?|JJnYp9+eU>M zeL9`GeIdk#lRhXv+z#Nr;kN+vtf>n(5C>`rE;zpZZ7#rvY-}s#& z1RvHTfJSdKLv?tD+e*x3hu=8tk3a%a4WOSsUJkxHF<5;AG)qjvvU`xsu*2S@*i9O*phhG=!7)CH z`5ocLMxai})N(Du8{F`rUCKGe}1Pfub?(-_^2;?~}vx$VKq4em#mTCO6R zqTN^sTwc?z3X>mE@E4#-?s7fMtnI^s8`1AiZ)IDvD(jiMmx_X|k!;&k3933^SB++V zx)7zt3A$%viGY=}rNN+l7pc1i6HJq6IuTQ5QvjaggcNxotZ!n)1Xbv|>7#S>%X+V0 zO^3Id23#O2eqAG`I9x!QS3&B7#3PI|N^4XTb8w^C!15FDO$zOZFV|lRvpV3AUf(qo zc!FEf3X}4qo+?>cYXx468%f*olsUDQV0yZ*Na^PA@x+vT;9{quS1AHP$H)dsz2M%$e)}lZ!e+#KcsErt~ z3i-qZe(nbB7hcdoY;NjdM2l8)>w-EoWW)1=zkGHS!GbK4RV|^vPKrZ>YG}Y_VZa0z z@5_)325%3>SbTxpo;1AJ&@!j*6S~Gl1im8&c(fh7@Kreo|DLTF1ut6N_I-NEJ9huC z@gjTmA9yVCM}>AsNP=322<#eSgUxw@#>)35N2Up3q=GggU|+?)lnRW_|CIP@YyD9^ zy{i;KZjU=hS_>nNQ)QMvX3^)Z!7N(ng9*cQA&F^Rzh1+WK6gPRuT`gy7gN3WCUv32 z_N>g%FpJ?TIs3}7gu(?zyW$3>icX^g1Il+IN>k8pRkj7#rN3^04@_a^G~OY^sba;N zouZZVR6iS6y}3S>h2qXQ=TO6c$x=U~qC8P z|IFe399BpEs{#ux!+laxnmidTM?Zgk*ByCk$mc@u0(I_*NWrJwgi?u>Jz+XZol_8? zXDRI1_PeQ38NYjai2#>&E8NE{-cXDkX?LJNH>CSvZu8@fcslY4 zxj9K2Vpy)?dB>T!$wOnbthBt{wOrDuvDIBs?{qG?i19tBB5tHz2dz`-IKBgc6jcI$ z(w>F*C$V?I=g$idcAw_On>TjGUGx&p;$Ax777=qPPunk@Y>1{Y4`ZQiWu(_0Cx6^U zWp>)et25eY>1n~i-^RlyW$!$HChRq*W1E`rg?}|DYOD4aBBYyMDg&fU0S?Z+XUe@1 zADn%xTNCJV(ej9rV%Bn5ubMh{SR00BW|tN)2S6Q%*V;YxpNWUSeBVXI5DoAz5J|Bh z-omx^fKA3%0X*2AQ_hR`zJ1ekZ*I>^y+D*2Mtk68_PIu4N21MNk)OxeWQfBJWG}s6 zB^4Vojvt(PI__QAHc)%`M=x{=iC~fLYL^QJfMXkK^aNk?BO#K@;@7X-cwHm*cKb9a zoQ2q)R2i<+Ra#WZE03)YVOR`oIliOdL4pQ5uwuz7fyKdicr<szdL=6dU?r{0=)xO$UKd?X;Wye(irh6$E*LHn)wUn#)l6I zMPcUD=dy$4#}3q$178h5AcFARbIz4dDz5KhJ3uSG7UrB0!xS+aK{8*g@55UZ-P#*0 zwwtnZoBTJ^nFb{N*a%3mmp3_UqYe+)-T_nKsCY7sMZ|WgjO31r5G9K9CDCh16CfHKGo-NU~L77Yz zo6?%gH$_=QzkF8j4IHf#dRvC`bKg*p7YnRUBphWFd`fA)ZpM z0vqSQ_795?-V^93JTKQ-dg|{NyXc?L|t~Y0WRHANS%l1V>`~qf?rmx&HKOkeq z=SFtQ*qQ8(U{7xnvF&#kSNhS~ z*>>qwf3*8U(r5_MU+HRMlOMz8INaH5_#Cfj7vk& zD_sl->rMOvgt?+JFF%v==T((yxn<`Z)kvOvU#X4zApNqb`CL!J*&Of7u*WND&i3E? z+Vh;rvs?V3`-U`zAqOYYFo+DZ?4LN`rKm$c%_i&<(Hmpv3eQ9`GFi%r9y;Ol35v4e zYuP)xqrcWMrrpN0ha9&C+Q}Q{0ike-W{0FVVrw%6bkjpOL@CxCzNZTupxnju*A$MV zrsGSeGe*=NPy{3Ac^hvB%-(7O<&(d{`Odi%e>yTz!#EQp;GW%ztoSuj#-AQHekmSJ zMFwWu8f2n$pM(-?1;y?6z}1{8a*kkyAYaR>>|1TnIYS)E{yG_HA1N~7%v(B1HWq}d zWUzaoIHsB8YipxhPye9`fhy5AGJvbfgReCGBUFxE%1{Zt(OsYNJSqukF?nn<@JDOz zrV7sGO-qYWXa_zSh_07wCE?N6{gGB0aDR)Pjxb!&n5|(oPTLfnsTQkotqtHCoVR#@k?d5x2|A<)H+#WJiBL&A3*qg4M?yG--NK>}Z1>@Oj ztSG^LNk;LvR=3KJb>@4IzOu%et%1xieoYa<450$X^>w`@lDPiF-aJ(34Ltd;H2HK6 z8NO-R?Y|Z;DxQ0;(1LV5nOaYou9<0tsOs{m395toA(8|)lz+XeOgp!gRRfKbJl%0y zBcoYQd68WgB>(E#42P|;ZyTw|T2igHe_ejUI&d>?^5B3&2vM3wrcMApq08;=rtkM$ z{u1yh4@ zy9QSIMurMoj^T?826#euo&c=nERm5wEzO1uP3!gAoH;zH^byS^-|=4hIJ~0_3W? zYG9vz`9XWkSu85aVfmg?`mcE{%v>bwm^VXPW1U>^-oqnAypP`)0SxKP`?&I@aS-{# z+RL}a^7t-|@Ue8TxWIO7EIvbf=cdNn-4B%lbzB4<#w&3ln82N>J&$ty(6CR?t|-H< zhXKRAr^f^F3V*1bhq%KD@y=Gs4Hh}9g^h(D3Zb@%9AbjErcxI)V>l4m+*77XtiM9XX* zIm1K1483b~@oYyqAFP)hh0yR5v}o%a9#spiU#pvk#$GtbXbq8XmO8od60ySKuW}?4 z2a+g4r;NR}2)EoNtF1`2Oq~sFFJstEjCQ6Qi^6-l|J7g;A82PAls-DDmW>@Foo+gm z&kL%Ubel{`Iho(9G~R9k2j(u{dr4jv_Q&I>}DFjr}+`1ZyH}a_-GPf7``Uv8&Il79(i1 zb0d0v99Pt{cV`l%#RUBs<$)vFJ;IuN)30-wGD&Kisdk3*AMfAIS%`_*o31@sdqMc{=Ff;MfoAgu zQNZlUB~ca!!$>ZM(Qa)T180|5LwacoG0HjcZ`0a}7eGO1z-tCC)B^e^|rXH_3aW}WN2IOdHSuKz4~%dkk8L_gug?Rf#-H(`iwyiv6V zMRQVeeAZQ^6*5;7boL=Bw^$S&7r8ckLVMRizEZ7T;f+7$G=3d#WwD@lHNY^^H#)Q7 zkg$c9JAERRwK`8NY}~sg%(w&E_7qw>Mf?YZe%_Hw-I~|=sCina27C5$b@c30P?a3fp2?I)eB)ee=Q&d}eGdQk<6 zixh1jMQ{`=qLgG9u5=lkikfO+5XGrvnXTLaj&P9zd-VOgxal+lag4fU@)?T$DF$=x zI6$u6mI?v#;KxZnt4Eo(`>etxdBuf}iLyxWBQRZgN9zG@f!((G*ir%^VqS0pNxmP( zN@67>cecVuVcM>FTIr{p3e4Ga=cU2H4BN31@J1UgBkJ)Zj|_{oUTgkxR;YCtkf<#T z$6M(T3-0u&mLY}{RL}An&?v|Q%yoCPzUHAqH34hN!_hw^a&9Oy{Tz)$KDBJOej!U9 z)2lqoBgT$^!m5I$KETjeB(^F`lpl#k{wJc~TbAU zAyED;5`-}Je&Z=+dLt2iO+F6rtkfTg0 z{63Ho%9>|=lc0mpRrMWp`7x49D$gkTu@Vj>QW$Kgc~lUhn)RzTnD(<-POv77V98O) zqC~L9Y!T_(Xaj}(TIA^s1S7b72I5M;AnAKuiEZ>mSY>NUZnLIqTc2f47TEVEc5vKQ zmNL+XIFDT&D(b}OJ!Miz8|v)c8aE!hC}9^?i=D)5I8s4>(un@bb>0@QI=oSpPLvA) z@oz^^=wdiMdCqZ5)lrWCTaQAyw&{!fvvlLveeT?F)MYe?jeE}@Y*Hvwqs9Q$dHjjQ zP)dGX^MH{O^SE8FH!)m>9SF*~c2!}DQbH>gkM5EO8k;;IcQGg4V#Yy1$#&Iu1^Jd^ z$vp}x;KX7gQEQy}VSxzgdSDHWfw)>R7Nk1;g!M|)U@c^XYrO%sPK}PKCAodS;+OBz zkNiwS=nLB+ev!^P#&li-92tNJSS%e;>w8YNdnHna>(+JvB8;O0!rh}^2eAgY4e;iY(wo$P z@gvrpIKc1fSu}iTY)|59sA42DQ&QPs8{=LN|5GwL+%HRm*>9Wo{PCB4}cBn2Y3WsG=7_G`viM$s%;=adr zu;-6fzy-SO)bBFSVV7uLLTa&_Zd+}gO|fmEk73LA$P5L^b->AXJ1QCBSHtN*=H+Xq zgg6apS*pcv;xd|Hf+{GuFWqX2XK}+~UYL9l4U|>DUV4U%qSD3lrBH)$!xxXU7dS1U zYDqxmpOnxv8YS{bfA6098|xTbEzDScIt0_la9BQMROrDMRiODl6koJK@qf+Z|9@nk e-)Q{e?z``UxWtE+SHBbhfS$H7q6+>v;{O44aExI9 diff --git a/hsfs/docs/assets/images/hopsworks-logo.png b/hsfs/docs/assets/images/hopsworks-logo.png deleted file mode 100644 index 36f20bb120bbfac7ec822a7a5094469f75754db4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20421 zcmeEu^;^_W@HmLlB^^fzigc$mDoS^UaCCQfDy?vIs^n2eHwtn`gVfP|oOE}6PT!x; zfAIbB`+A;xZl9gqnVs3!?##~4ygsR`DiGk(;G&_S5xjr*S`!WJF$@h2T@f1-RYM&m zdXDxqs-c%52v=8GV&>5Tr-e3waq^ z8EHd%<@2`z8gcU98C^M%QMA^4A`FrlvO$~ft$S=mTR!v;PDQ^;W#PjkJHz>du_D1k zzcCHo`H=MATnl5vh_gREwz&8lZJrKbVKL%+Li(>0T_o^h7|yTFHyR7p$DeW|YM+x^ z{R3RS5sg+)cupM9_mbG*m}SaxhgXE?FC^6WX(E%LKpAlWjP)ctBGOXH0Ovnkr>{k$ ziQ}IWzta1(c=*ZK>kE7B6D{t4-2I7h#h1@|$LLK6eH%!UD?#$*U-cN>DpfS^XPKC? zlzrm=twLX=5>zSDj8`AZSD1yJvixJ9;7K>1ZGomT{C(CrkGucGzae_cielzt_eOgq zv)6i!{3pgxC2ZJAkG@={+U_Ew#Nbz~Q|A8=tV!<@i(1Q3g+p8y$lpLfKS=)JNL3fZ z)UzH(Zl_7o)aO3Vm|yUD{15O11aw`Ih;r8<>-7nTyv&jQ)>ZcGAF36zNZ_A!*<$r^ zS7|A;*sn8vUvd6*v5d0={jqp9W#O~i@h8_Z>p6$vqrZUK*9a;{uGp`aF&{XozkPjq z|0US%zYZ}2d^4hYlpW?t&vzCo9!!e|{&h%3aXTZbSoCz<>r1F(kIla&*@r&CZVu#L z#Tgg+0#Yy-{x|;QC+x5f4(vK_w3CYtG#+XEL;D3ab=**rcd1lE@?ezjZR~sHvE1 z!I}-o43F`An*49ZzsQX8UAk92GwQdGc?S=;%K7i*CjkKutp)`LIn`zV?v-dCX+Xd+ zSDvXi71o*D?0tW$&@`R_8CLpSG{u7X%jgU6PC`-YYhCl)#~t?FpGp8YB9ntc6;th{ghwakCSFjuK?w{X0x zDBA2GBHnK_hGM^piqCjxe=hnz>`%HO?J6u}0l*eYRgz>Ux?M?B_p^cDeE;SvROG=N zZoMNLjyM0IB3ZGz3t@BGkY2F&EzfsCn8{8Mbjy#Odfp0wRU`H zzjj#AXY)ph|AL91vBQqtd8RD5OL#j!=Wl(L`}0zE-eJor`Z5=c4<`xyM~fKH4Uvd1 zGTh}+8$NKc%tCbk9u=54%COIjHp2N&P}qbM>=PoNKdk9)mUI%_?i&m5 zh*)dB->h|^@$C2_<@FbihM(xlsR9nw4LsF&qf7kd%^nLVlv`fwA56TEDa2n67mAxX zew7{xB#n{wVMd>I*z$<>jkg{9w~JsuS=ASZl88o<#e+INmw6H;hB%9{1i0VUt&ZmQ zOsV|0YsA}u?5LF9eQ$VXW%er5e(T%8A0a}?o?>qh- zV9$xZ(4x5q?HP@sJ!SZ}Uh!jHIi2~ifYA4kDnPJhzLe!7==y<-4|1#KVpsYvk{>8$ z*BVt{9{Q!B4Z^gQfSTnmX#H*_YcA4W+%>#Nq^vC!_*Flxy_~L;GBB8tTFwQA52_&Y z`NXA9Gcns2=kYmAO5BH397Yz8V}B!h-4cs}8iqZ)(5(cnE({P|YKYdR^mIsdyM@(~ z*&mCRw|4RQ_2*IN7eQx$2@9z@gW-U@wEg!!pgU9g34(^bStdZ7o-J6j_4wr~Xf*Cv zKNyS2Kvebk$w%PM*N&uC$zoBztHh`tk7omb#rrx<-cp@r%P7~WAjs!w>A>dr52GS=2oYep?Iww&C}~= zNFNCqHZU12I#8z<6_XLA$UhmLY*JnVzm?|K8c=;=K$T&7#nBDfm&1oai_k3k)bAPL zI?F_itslhz906(T5~r+f0@Qkuhd)%-twX@B`$4H5gGC-Sr%lK`?(hB~`>RNLa;SlP zMX;0ZJ2>vIEQ!IEb&vK19o$v@*)jMzSvTL_fu9+?;aN?Z`uW%q%Oi zcyjm8S1%$g%_tjy1reW&E1zwtpwIA%s0XwZ6PQ%`Ge_(^42U7yLf$9tTOM}dcs{6F z&Ua#Ie+uRRJ$~qG9F`1ZuwCH2EKMFSFF%Kfj=bI~))1)3=s&no7|Abmi9H~L4qmoY z4>#n_5}v;pAo}S;V%#E}=C&x!pNCu6wyQ*7&?o~oZNj75BS}-J+6xw=h-bi5~!-u^&mx?i=c%Bsoltw2CPkX8-3+R z(&@(g16~R^(V#A$?SY&8iYXXtI?AK|!Z zdKqPLmpXByr|8xNG3>Kr+;TT+s;p!Wo+!Lpa5XV6aN|0nPUPA^I$)+&Eod%&Uwd0s zt-T`@pq|(L@Y}7Z#y~@Q^WKjvvmW*0b*j@KqHS5!9PhXMdVZ>W@ZG1fWRz2CdHP8# z^yNnMwu_uSk1~%6gQR*^1<8)yoKGl&(K6rO7H^fyy&zOys(3I5`8PuFDDmYVzSAwsR+FT=MyOkep~PSrG& zwPkBV!t)?*TxF7d70VI9?w-vk(%1=5d1w3=*`Ke#7wS2GvO^L<0 zSn}(H76IZx37q+XFl{km`ag4P?8y_@=f*L)VC(?g*~{*P8lT6O4>VuVy9OOp2Y%{L z4fS+CK`CoIc(@cWcQ$ZUONgQo3)2;Q#@L^+=$;t3QA#Zye9=2W=Dv5&Ew9wz?pW$E zGBxwEI2Z`LG7!rVIViwe#F`h;dKi=2YR@%0Pq;5fY>VB6I0uP z%#0q+ELv*D#z<&p0+T9TzOTc=D%k?53N~Kf9Q$M!6Z)t6Wy}m&o)`KkvfQRO2 zG$PgIL@DTtwa#q>)GeYC4>csClQ#f=+wL!lW=+?;`_Q0GJD50OD|nrFMv(pjBP5Nx zn{Qdt#7DA31M#n6Z~}k8hr-3shuI^^Za?-L+8Ajb&(VQ=y+dqRH$T&3*KTY7C&QQP zGG|nBysT6=(p4po)|RJKZyu9OJl0e0*q8*qIiSiqCR$vtJ~$xiu#f++TQWaX{Mjxb z<^8DY9_B3;(M`jSk7C6iAzfKfinQlot#j)Mg~)PbQ8!w3nCa=^?sKk1IC(#PjO(nkAYpyMU`=SgZT8v_QEUD?|C=fqk8?ULGaPY zwJv4=9*nb&U)9X99NJtKho=Lg8W8MRDNJ~ZxExvZ4kYFst{`buo60!ab@s47>*&WV z{Oq%=_;$HH+@x)xH9ARbRCEXk0;*BAvDz#vAGu zJ#65#aD0a_PDgie7OBU>BFA95pv!*ZKD_ zKrW)CIvWWr)TXv`6S5vJ1?*f0k&h=BrWZI@xGH<&h?G5YhD~j^T$(7}=(M3fj7-QM zyID_EFqF|aycHF0Uq~~#NXP?jePh5BR5$-%^J_R&OPJ`9JDA49%XFWM@@VU;(%GbF zjr|liOz$Iy&eDX|A~>Ce-5WbdX=Z04*!M~*p9wmPcW-a+O3X55l@);m|l#j0|oaA`im!d2YmX<`CMYR4olJ0)mX<)Pm5 zWY#D7pw@n9hFzOfg)1UwY$V@EY{ZdWI8ZuJLVsz6Kx|1EwpWf2POrYvnmxY>Yh13# zqje`ggCH?76O1QDtJMgbcD~&7_3)uq2!o`w=LB}B3Sz$KLc@nMJ0`!1-K>nJE!+;& z^Y!}4pXklyL%Y(SJuSa?m0GB`NbdKaD~k~oFL5YSgFVtO$)bwdyHdIjTAhBwRKVrH zr6aqOM(rSXsm88zQSPAAtly>X^#rkUx50IbFP33NDl?+gw(UV*bk1HJ=Z>GH8>$okjoV||+Wj?^0PUPLRY4see+kstlIaRXBbv42r=5&Csl0-DD9D2(_-x7*w(P$~jHHWpHP<4Z_F7s&< zHOzE?(Fbl3!^3)|L_v?oajM|)XSfZljv;`ilp7zk;F)%C8 zWIl;&&Mt?nxcEhct0UDZ%}E5QG>>-TPod$T52}8LO*({q`~3P(^_c-}6pD6|EQgZL zVv42eaHD>onU%J+Tj(=m{HsE>d8d=02^X!c5Dc09(ghxLk*Dzn;hl3xrhdGQBm(l%f7S3l+{ zFN8FuAT41@aIpP?OcYsKAiECZbz}Ov!n%(WIXnA&d(zlwx(L@p&H>gBr_oo#F?8~} zI>Y1~;T~CrICq%0qn{XnK1ES>fdW%Az(gC=#9h0yD@`ApUX@{X%Oa?Zf4A}l5w7?b zO}UD0BDOUSZclFl>_YPg%71o7j0jKR@CD|Z%)d=wj~C9Lnt3*~10L)#NKcFWsk~ zCoYCf=7EY1?R4Yi?Mg+VXL2)Pvp9ihZRklmLS8q;(N3(st#)h3LlPR@p zd4Ku&F^plz#Dr4&V8s$3)+3pYwAX@~1>;yJB2VqP(@N8FB8 z|29;)$_`fJT%Fki(F+#}uES^YV}AI- zboqycf0Z37h$#fMo|+)m0Y_)S+zB@FrGV_0Tva4%`r5#3ci*wzuh%ZF{gdv)L(WmPq(lu4&f zHVY$6pVZ_1S<(jrHo9}QZ?dJV3Z)!lX;w1`%`ox-mlYI)cX4dbwUY|LpESE5>LnmM z?Ybi0;6#vfK~)qG_Kt&ifZVFEVg~Wb)=``&Fsg!94e2v+el0ah#07JrfoIDXkh2Cv z;r2Z{;!D@1&B-c**xx&k&|s z8o`sWrwD2FC?qRyyb81c%+Mmp+k`Qa(xb}(CqhN;2V0UtmHJeQU3>{EFE%_>f%BU5 z9(Iy_17d(|tn+)|fl*-a%&AG$m3GS6YtiBeL3pCI3t!f8N!S3NXOY`(q;YMvS4#T$ zhdtRxbOcUP`P$d6OYendgA2BVJ6)#V5)_6f$+((al{k3K0|pd9H1MqC9=?Pzrm34O z(3*0-aStR3Jz8V_b?kHJ?^%vKM2Fb@;*5Zarzct_Duz=QR;P4I7$D#uMc^Uvwbxsq zp(!rAYYf_8i)yV!AfO(c%;(gV`-&j%0MT@D+3RVjq&kxV} zqU^NAN`myFi?yk@Xo-S)C%qbQNV}pU1QT_t= zv*&y}yrIQNeHu;2%LGr9SKDZmt}}lVM>>?Y-*Xy-{Mq3xy1Cv4z8)Ke%gYSAR_r0a zKJBhcCcPH51SFEX4^Q}x{qE&IV!V$CWEd+!ovQhQjKp3_BhR5q^l@K(*>F0**|kyM z*O&)W@=%vV#97>G@FWbOTKK6d=0)eD+79I5dhzK&=XKfCqVG*9JEu$60q#O_`7Mx2 zgkR~oG22<=_O{Xw^x>d2!y87kq$q%Qgs_66!C4YNOZZcWzQNp%;^kK3%YS;W0K@h-(AoM0=Qin=*A zd9pC2QDwu1Wy$=g)*6L>ZUE;}>WJToGcIEhC0dry9K}jZ!2X#KIP}Jhm z5`Q{;`W{|5qmStyVxs?O(R%E?Xc{K!77enaC8XOp@Zo8Jk(v%wC+smPHSAKKuA1%f ziIB;u>5|%n{?L=`qi;~(L|vT&)o6=R2?MLs(X!%`#cS*^9vCyYbuI2mhG`sO~Q*9B3U-(L?J zChLBd&6o97hfv4hPq$S*A~g~)k`c-|1uDR6$@`5}9qGy`q2Nd2Pw&{Cun{To9yH@H z>&#$=mO_DAj*Y8S!%ATic7pzdl7Zt?8OOwHt*k__Z`2DRXON#^hripPHU+-OGQ%}j zMA-MFGLQ&voJ{0o{4(}cm_jK?RBZmYL&fixhpwA1i3BvOeg*ezYO)QPp8!R2vi z2^)7U8?Ta&G{n5QZjaCM0(E{;H?9jLWKuFoAC6(~YwoA)&b?m+d|J%L)-^Qk*Wc$0 zbvg<=)Fn>rHRr+F(=O)dOi$4|8Qfw~Ymgp53wTD!Z>D03_a1UlOJbECOd%XWnSX#G z(<~T%lHgG@ZOSPBdAmE`p;+IK ztGe6WbpAi>>2c|A>*h`v4+2J)2!$~si3iWoFH4@zEnrFzC^+!G_HwSf7+99gek@=6 zd3_*)3ijBkQLGUw%%qC81;uxBG3j~zNoKHWWx}&osXbDHeq+543R_7FJ~EHakIPNf zUlctK_D4X6@Jr7zLRJ%=2L>SjAZM%e{f=)RlgaqjJBQnZ5#9Gi*B3bMaA4y)gspNRkHpjP1|rtOO83yr_joffh?8>h2LfA1MFHz? z%a?$RLAvKfdl`ce`-yb{vZh3=^X!l3^7C#>FKWklCI=w2eusd$1{H7VN8fFfj@yrisV4h&-`wo$y`!k`pOOs72U99^|fd|t!h|p2m0l!<$ zvG0|Z|3H7jCy3O(rJLHqPaxyFSjGMDSMRC1%R}xK zS@@rO(V44LxJ$nwN7=f{4ZjBKI{q&fVDd!|)MtIXoNGCiox-sj!mJkX6-pr5Ish5} z@YR#qDquaYHaC#9SJR24CAnE#B$!rjZYb zfMF~ahDVjn2j!^dU#Q8LFo!#h;vXluD#O1|;-4)L?I;z&70L%DSi4?|aJ1KP4x0?( znlceyqhCVwXudBz(XOpJ(-uCHLPIpTb-|9Mj=SgLjI!>TLIan)= zl1l&O5@iA=Kt5jSVpD3WftTvKQ(&hI6bTd`p-ka^?bPtv%yxI}8;OMc*VYxQj^u=Z;*bnjQQkA~lPF2HTP$j^YV`2{XuB~WCjKcp9(xS^j z=q&?*=$%6aK>ul?Nz*%9Lqii@-IqGrAg)x5Kih(c1X=rK)2YQ(q9HCzdJ`sRt}90o+_w$ZVVqVT?xB0o>ILY3^`HxshKcUOU^XG@rYe*-#cXofiIkL;7l@vY$C^xm8hae`g?=fGAlk3`(y$kiKv%0GQ8YgE z*$(8+kBADg3bskwvdP)NrmhsYR+(ftPxi+ih#7VCzE=~dWm2a-ZG)iOYJOKMBMNi#S51Ce&pT>0O!g- zm&`x0g;F!r6MneuuJOr^#wzz|2g$u!EO-n5_aTj)A^U|7RsYHW%+JpGYkdR+1FTJG zF?|$$l>mqcXyH5jy)L*(FHxu~bKW7l@OW+zdL~CW2h4jVW*vC?f*%B(Gk3yWe|zGp zKbVak5NB`5&ZE6q*JYMn4iRtC%nZ@ayT>8IhAqSv)^4}-glV(z<5j` zxDmqv3oFkXihajjyb-T7fe``TjJg-85h3}0&kWbrlbzLvgm=SmGYeIJdO=K> zIz`~~^)w%3pH2J-K>Pt@^K&~i-eDujwSrZQ@Qx_+_9t?}8^}C*F2)ALM?@X_8 z`nZ%ls+~aa)nQ6MWnNLeF?HGe=2!4D^O$LZjwJRqU?^BJpM($=D6vaZe!P{NBfmq4 z4#5Gzj|HNA$23@k=VfkI;d^Ltg7~rlX9FVi^v#}?G!E{}lHDrm*8VPYi91k?Q_7+1 zyx2UT^DmNkuQD2==gs0!C%eREgI>NSyq|9|Z|2k(gHSGUB?W30f`;zAvX!f6NIO3( zV-Es`^t^W?8_V*|x>`cQ`1v9)RN~MOK>KZNM%*;*}~#4^$4cf}s3Elf{VGt-^; z;y+x^OijCnV1hJ^spRKF-5i^SfMc?YYGpV2$enVCLlqaMAd4B1V&kW1;LQ*E)LaQu z`Fh$~k(M}Hw^(fk@0?}0nPQiNw&9kvR~A?=%XFOE`cX?2O`A2@p%>!~A7Fz0CKK!O zSLG+CsTt9_DX24|_{7DpwJr_MYCwe?9wxlr@MelCu-{xfk|RbL^MT2t>tVwo>w~Qb zK6eRsj9|k!GFa+ica%OP*j|b?@LRjC^RtKk2|3n6w<(c^%2x`7X`|(k8REnzH3X2X zw-ofVgn0}kS?)FtEIKr1ESxt?5wIxjiK62tKFZ%k)Xyp1eLd*^1X*4*^4m(~fy5$HymMkSg zh_755<83WW%}@-3&MZ6!zT5kARHD_l_uKOE?s+&)nM0@|$|3p6N%Iv?6TRvZE6NAF z4~IV7v}R}h*ew+$@L)HqC2~LM1mQMyAcP%Bn~z5C

Y_~oqMbws!kC_?Unj15B5h?RgtGa)YCk&i`u|ZE zq8NOHQD|@mP#@a3oSSSvCkf7kFW0VZQX0q(RbYEhm|GQQbX5>3Bf`VKJ+{=Qy|qW0 zi+R;6yu6x_HUGe%7zIk&!(J_ZcXAV0WGqA$SB+Dp>esJvCGMZCLG?|t<5)gspdICB z@9Haq`!aX)y_&!-q16;B2Ey0zT)0=^K4P8w_NKz1*rv28;Lb;|MJY`LML9vR*VoaR zf>N3r5On5w}*a#{4F>me!9>XZ_N?9THJ*fj6N02%`fe;NqRWQ5@F7v zCyThwCsdr`zCkNj2uhF_L2qwjs$wM){PU1sGNflS9eD69zG``HfHA}54)ZKy~g$*ZoWK>ymHB@Y2emb9Tq1x4&SoG;%cj}2VFZPls+CysqL^DauNwG-}Mi%yRy zb6Y|neqY3keUw?vUP<+vq+dr~@ow|ZeAlAY`}7 zx|}{aI!*z#6U^VWJh6Ng4q+U-Un%K96RDREKDfMJly4-Y_MdVjO-4*c?0$}gnC*fn z?V5MSWTbTEm8;u5C7JQBxAI(X^&)}rUeVmzKfHbBRgezu;nI>=N<{WAFihW_O8ym= zWi`Kk?{Ve_PAcKJ%I~{^V+B$OS4Zg;j9TIIg`Nj3vIwO&%JFb-@0K`DDY^j@b zSP+({HoIKzf<8@(2v8f&B^RZH6S%vEhUT(GH7O|RXfDu;Os zCcTy!O3fL?33`d#S#T(3cx=bBJYP*pXq6o~cH#Vy5m|7WE290z+ja`ip`}c9>(>9v z#p|LzDv$n%fP-*Cb9Kz#O0?^cDts$7A9K!h zK6_s0F@{{8H{)wOKX!%yAhOzh_^Y^pBP~v>sg>q?Z){8d1L7{T9b>>+j`Wl38V4TO zJgmV`R-SQk6ZME`G zC!4FrP61D8$(c-^62U?wKAUM8^eW{ASesDggm-n#IGLV_LYu-~*`?>tqU%-sqVrtMk zbj3(OW}HydqIq){6;fcue)266fk*G@SsWBX=AjW?$yBUYWvFUq@?QN)5$eLGof2fN z=oJtNJhR{ysV5+GH+VY}Wfh}t-M5u$Wl|24%S{^5BKo!V68YJ#fPRi~#~|;EYANTt z5ku$AhtyHg#)4fk->JAw6{xD%%2?G#tlf2E<^aI{owXc9jOb~=Jcsl`Yn$d`Dh+CV zF%KSmsf=HS9*4g8#kU}YNCt2{O}umdqJdHtp|QpKo=)8in#sU6t{WNJc+x~B;o`9C z*TqJJVtacm7{ze+6xda2BkJn-#Y?=PM?Tr z{^jO^+*RFhnSTY<2Y-YK4C}!`tO?5ReNA&XIY`RnCmjjwijy>HpteNaEtpnJunjtk zr}8;e&v^`!YCR@~0PR|f*k<9PT|*!GwLcBpHRcxqt&$%Uoy+qvJ6m|L8cD?zy-5txUl&}}Mk0D2-rm!j8s-p#Kk98vo-Q!GlQ0u}AUx~T}74RS3jsP^jC6-I}Medc;i zpv1L)4U+Lj8oE9~#nDJlwPx*C>`Ac5YSn)nNrPU?y2hpGTcDdYNU+AU@L zunmZ(hguY#yCCSqHt;#8c-jcl2qk0i=7Pv_Zpy&jP_z4}{=SWVC%q#T@=8JU96ZzW zEE~6ktb9XI!rND_4*NS!NJ@$^Dj2H@8Fwk&nthecT}b~gmXq?g$8L$;BOPtw=QUDF zIa(I9P~}Iah~HFR8I5Fe9`_40pnoy2WWF7DrFJ@K=+5a#K|Q0zdJ(At6_WpWkPz zO!%U_era^&PzyZQR!-^;I@IeYz%lp_Q(z&}L)Sx6Ou?B;F2MfMkGsBS6XMi2L{$&uZml%~zmjyfKd#?ifONp^WUU=z;wuZ>zH z@B#CUAvv$qZnck^7a)30;9BI4)c`nAAhOo+^e~FbsoiPl>MX=K=EBgj_C;LyFbqL3a6bL)MvaV>~@N zn_rVA)${c=nW;FMy>>l zyh+f>-N=A+gsLKL?c4IbA2*?eD(Au&^8p=aHZ5W0n50PBCKsWQd#|K7t|MK@>a2(R zM$|!RQL2V6f0kx&G`|T@ox@^lD2^LEw5Cc#v0fCh;M%mvqq2VActHSt9gKVx*LCtT zq{O8y9&#JME#<$=JVd?bTW6J+kd2ZaqWSpkz3QUPZjZhcGK4b%B6oi) zO$hZ>0UmVb%a&TZI|f3qp}1Fd%Nw1=hCc=C-^2iIwZ*4zxUZ+q0^3qtTah4s2&KRFps_^QjRy_}xn?pqoV$Q?>TUz!+eN6|oN*y}SeeDU*(1N`t)Fm~2Fg)k^ zEN#neaU^S~20O3D*Ugdd_C@|z%81MHqqs!mpLvfI$j4n3#>vqkLO>;B+m|ZAgK;5 zTd>^mVYn^&F5>vk6vuDIPX!}h+@hh~?vJ^P&(*IMk4Bmz4)zk1y}(PM4Y{#a@wA<7 z@@8)SoHr|mYLmIj0yJh)R7Qvf>VTgGZUJ3_#)+q2>Wg{|MEV?S!g<)f~S!#LcD%dH1X&~aEEXMBRNi^Eg zdX&S-sUmM9*b&*_A$a3jQQ(eW034xwq6;J4RGEgh+wPKtz;@2@$(+ zX(a+UYwf66KeEuvai*CpUh=(k)iIwS7oB`ebeD|>z6maLd*tH%R-lWa-vhL#{-}7n z8pd%^rt+}tH$5^qe#4&sjR$e_b-$DI#v%%pHNaK|B6sk$P%Hvfnblf*fwn1qID_7! zp<&O~7ADA=kpJpjE#59fT-}EP8FUjtXNm+;=t*ST`OUWK+C0;X$<6GP9o~%uB zXP!U3X(|H!*m1UZ0qCh!(#R!5MDm?Pv!?*rCOgPx8tD`g9nKxR9+F6~`>grT>PKwE zkh%oO^sOVXmt*}D+3S23vNo4yECJZqhMYqh50@@`u9@hvbff4pvNqQXd6U}%boOSV zYCC=Pf&lb^p_(%s!bAS5ojWSIQ17ms2{1^cBmN-n*<5pGz%Nd2lAp25adVLB%B+Ki zRxhOUbTYw&LI_0tEe(X>6=DsqgLJv5Ovnu5x;k-mr5z)8oHMGqOearYEg9{;5=bkL zW_%SxM0-GMXr7_dJFnVdI56x38a`cYeS3S7b;oeivr|0!(D6nnNldt0Cm;s>`dXFt zdNT8*a_^kel3}vN6OcfAUUqfoG%8Y8BYS3ewQHL%{A8_-;^Lhb?DprexTAlmq^8ih z*8ZwKJI+@BS_g0b zMcc=Z1xRtx7Gs+lqF242uwVFwcVoV!-h0s^&jSZOFSyxmuGZEn9kW_!|GdYs0ljP3 z{njX5(ke|v@Lv9>BJI-IvA6G~=PAhNwu?8(HMxF!wZRGS#D!E;D)ENBCCi0|2=t9 z?G%}HP)qm;b}X^ip+g+ZFCD`7%t)Ob9g5SSREa7yi3y!r3)2{w(_Z=6alPW(_xCEp zcbU~EK?qxDoWajJ8ab4v!S!2Ss6I~(5Y~9_|B<{IkoQJ z)BVJVD?puH1}ZRt;k8L=ceH`LVKYWzWt)U~m>YuCmv!H+&QXV|jBkW!KWgY{bmbhi zj_OM7_}ukc`U6YLJpUMJd@rhMX$QxGTl2LDdbYbE#CqF#g4zrCkV>y9>8a4?QDqU zF?3tEzu~jWB0Hi>7+9?AeZpz>r7F*M+GNQ6V(175EtIm3LhCd17`Y97nQtUTB{6Vt zrFnIrL>(hsdD{GO1>Hc)s}0O%;{`1Eyem-WvhAN7CF*TIhmwcdDt0TF za0N6tWJ^P-gknbMRKVP1MYuukwuuiHej)2xIa6uKGM|LRfP9<_!NWs>1n-sh0EMu- zij8bN9yV-F088hJZC07vabo;I_yqy_T-9lmUn*Yo71DGobdja~c&!e9#@H15{o)Q} zmj|U~7tW}_L5;VGrqkLni>cOH+F;*n!KUd+vdd=Nvn=Q9K>>!ge z`E3~Z<1 zOAauOA>XSYW~z-V#8Ia6!YjRRw-bf_j#>gLmHXB^pY1WcyzVx#eo--|>Xc}kIuv)7 zD>t-#wx@AEKf98OwlIENp`rsXM=FBiG|vteYR@KeCCL2ZJ$v<#*sx==n}SV7N)o-n zZBuy^4GaBTT(r;($)@(7B6qG}*=g36%=(s9KifZXz#Nnl@y*0#O<>B6u|N>_uk72_ z0F1r895?_5rwA&I21b4B9Tc3Tqvc!nxRi=+y{|~_9*1Q;VRc4`9+0YLE`2V>dFW`L zJ0A*Wy?MC_(#&N9$a6;LFsO*0YjJ|L(9wwN608t#c@?cYA5d>m)vQvlfmV?wzRKlb zmh(rGMNS_l8Tz&O$kv!V5m4`wKrkLwRYTRRqL+c#V_#OJ{2!S8;`#71XSmiyFlrLL zki&(?U0|5pQ*|n*@;YlFtr?SOE$gh!ul2TrK|Sm>R>;l!n*}iU1L+T888c20@D=s< zjxTugJu>qn^#l2vpHv2)pfNsCq88AZZIsnljMCSsAcA=dJq%AZ?|1edfZKjo+ytbz zJNr&`G*z7w^7 zo_k0ZDpm3aO0|+P4`xpQdk8cuCY-jRTvf%<(|hpk12FheE2Q0gS+Lz$x}suUw=%i^ z%yW^d_+gqZyXiBQrPoC|cQ?eXC-q=ks$#z605k82Hhk1UgCp@(DHCHZhhBwC2@!lMTI)O82lK-54f4h_rQCov58~^kR zlHXOx_ll(3(nq(d2SVWXiCxZ+ENkylKpd<>u=kL`uGKIXxGb`@IMCnBUFmC1gOvr> zFK^#AofybBN#%Z)9i}Imlghr^Og!Tk-z@eL4Na_2ZTn#RQFt~zISn`(t5-$#wH8B@ z4s6tcbA7vYprLbnsmCda)~l|1By4n&&b2Q3n>NPKU>e(u+i_%~wNeP)1OJZMS zY^!HPs+xJZZmQO=ggae#h3?vmPil5jd$DHkX>s^P2*D(90xiJ&GJS4wVm^G*kTb1oXBEmI!H z$Jy~`pdgu>jTBqQgI&Y%bUxffM_)7@pE#cPwp(W(0Q9^hCpoL!Djza3r|-%X^x>yI zwfko(qSh{)to=!vMIbh#pqjvL$V-q{HZd(OSzd!Fx9v<6atq`s!t9qOzSw)Fcy!{Th! zc=&3)lBDecbo_$?xu-@%?T4KMSXqj&ZJ0@Pjepc_TGn`TDnH;p3M=fdWYDI=gpGeGTZVV zuRq^J;17&`DH=HVLr*fnYl}H-jbRQ9okino3KVqnObD}nlq5c6wDk{`J2~rLio6hI zKauK`(zG1;YV#@4TJqwenAs6)qndoyGqJ&dJ@2VKg7UW+i-(7;rG*YsdoXslYQ0>3 z(A)rXSlx5~{>M{ed0CWTzbg}-XJErfLg?C~?T>+7JM) z7^(Ii+nbBooFk<{G%t%>)Xsr2GzX6?LL+g!SUpE4h#>EBRIgH1HSk6|xvBB5N?Xd> z)9U6mtgzO@89xbp`2jr=+OeV3Oq(*c>h#Z-rx&lQ7nlI@SfPcGWwOP`p_rL#ejWN_W>Uzq9)^9=DR=VAzaeD5AlaF0M4i8oiqm@j zCW7KT+ty{$y3ZIWo^dA4Pf82-`M5<+&+m z(skm)`fR_Dn;MXaSJjU{6dxR}RA!bM9z;)#G)w;}ELSr+O1if`_c{DrFyB@LbVV4y z{@`4WWTSb%_bpPIIJDXO^mC2G`>DJnUmSbX<@4Zb!A*ozp)t=*U!Z_}JV`G2OQyxQ zW49TH4rb%@0dpd0i((oA>{!1M`5*=vW+CZ2E?s8a@T{Egudp*v3(X6kJaw3%7yr=e zQD8>WYbK|2`xv-8v1(Z+ZrheNr}2X0+0naOw_?kiSsCe1hgC=kg%>8GKQ@Q?JLzbe ze;14o;Q0~ihB%kwH#X|dkIy(9TzI749p1WPCpAv8X~3>Hn_hCy-5_KTwugR(sMF~K z+PPPwQgIPGD(^<#LD3)G6rWxhdZXIa4CM_g?FZ4yrp}&&>Y+7GRi!_TJQD-y!yMM-J@I}^#qFVq zM&!sF=rv6R-KxgSYv%Gu`-Quc>aWh+K*d~;`vXot&7o5P0iBk0r%yZ5J1I0`gX^tN z1GU&X``tHzk)H)%SrkxGLQ+9}o}g4xp*rU_rthBVreb4PZI|N#cjWND28?pC7NyQ$ z|FPM=3?D^j9c~mrni}qUSsU5Zg52-@w1k6^lE$Yl4wdZ>3ZnMY7T1J((X~JJYfNwn z@&>0^%Yn?ath)o%^C4NA6zJV_V-5O*MHI3YIG9!|(4mT2DMO&EjdDJC zZ+&h&Z2H1D12%4ZPy;&|<4x(PmHv$;N zz#o?+(7OMdV{6oAnVt5?d}+r}Z@0+bCabXm$&q8v=y+XK#{(e$2*mig5txh9j&apFIf>t`~FDJtds4xDrpO|J(Bn=5Uz3*SNdEH4mQKm9QXI*9F);r)wtcFx$U2CAnt#Sl zFFR)B-`5?5@0@AGuE>=IW2UYqB^E?|tbH@o6HwAytFuh?>?pOh7cH}t)sz_pg=&Zr z?bt`7>hpd1Yfrcrk6c~FX?0dUU#=8!B74T=V1M)J;XzC26q;TX05exsJc?RB)k!|z zHe~lrH|jhgNap?DHsVK0;8;sQ*$1qoQRs2Xp149oYUZHohoK%@OSGX1nASuH56J=< zdW|uv_U5{6YBYY!yWnb^nw3n>`myL0gcx_^ws?ZQO)JM9S;}|!CW+9lggFlW!U^bG z{!}h@f^CBW%;o3wLaj`%Lv9anRPMy!b4@yTUpx%MOAI&B?ZAs3fKtpn?TWPvEF6sE z3pqgHg*qVlnH#SDHz+zI36F=Dhv>dnxqZA30Bt%J^j7ztGERKEeYUUkkp+J;XA;;@ zs-2#fAblre$wf*C3_FA%Ddrc>^RPqUjnJhFU+9%j-tZXu;*rPpYuRBg|FfVIJYqMPLO6ItHF=$M@3_PkHn1bu(at96qoyN=W zC^FNDp1#y32b%Rgfh0%+@YV&i+X)5mto#$NQ7^PCZuG#~ksXD?QPfdw9;fdTIRVoH zP6n`I@7%+m4sJ2~E+JsPP|xQ6WG`Jfa(^sZx96J^CPf^fr1$v!&AC&EjaszHy}WL} z>jrLNFW&lIR<4puTQsVVFd|@5ywL(lRT3iL$UFiwb`fe!)I#B6zrP|oW@M&JkF1{t z9L|Xi)U{5)sVXPVP8bkcXp%+QWFL-Kka&3Gk{IJKg6LeI-s%z0$r|EW8_eHVe_5uN ztR3jM8f#qoy5MajBpft)Rn^P3-F)))`xFMpAHh=V*<5VAwVt4q3MR1nw^~q!29jVv z6{^-Yilh2cdh-wPxY-I62or)U7lVpkJYDOLA)PPzUQ6pl4W6U&v?1EATK^EJ=k(0= zOn;A0g+1d8AOvJLXe#N*o+$6WM$%*w&Md=-Ho+G2Xpn LcDwqP+mrtQXA#fc diff --git a/hsfs/docs/css/custom.css b/hsfs/docs/css/custom.css deleted file mode 100644 index 45f87459a..000000000 --- a/hsfs/docs/css/custom.css +++ /dev/null @@ -1,114 +0,0 @@ -[data-md-color-scheme="hopsworks"] { - --md-primary-fg-color: #1EB382; - --md-secondary-fg-color: #188a64; - --md-tertiary-fg-color: #0d493550; - --md-quaternary-fg-color: #fdfdfd; - --border-radius-variable: 5px; -} - -.md-footer__inner:not([hidden]) { - display: none -} - -/* Lex did stuff here */ -.svg_topnav{ - width: 12px; - filter: invert(100); -} -.svg_topnav:hover{ - width: 12px; - filter: invert(10); -} - -.md-header[data-md-state=shadow] { - box-shadow: 0 0 0 0; -} - -.md-tabs__item { - min-width: 2.25rem; - min-height: 1.5rem; -} - -.md-tabs__item:hover { - background-color: var(--md-tertiary-fg-color); - transition: background-color 450ms; -} - -/* -.md-sidebar__scrollwrap{ - background-color: var(--md-quaternary-fg-color); - padding: 15px 5px 5px 5px; - border-radius: var(--border-radius-variable); -} -*/ -.md-nav__link:focus{ -} - -.image_logo_02{ - width:450px; -} - -/* End of Lex did stuff here */ - -.md-header__button.md-logo { - margin: .1rem; - padding: .1rem; -} - -.md-header__button.md-logo img, .md-header__button.md-logo svg { - display: block; - width: 1.8rem; - height: 1.8rem; - fill: currentColor; -} - -.md-tabs { - width: 100%; - overflow: auto; - color: var(--md-primary-bg-color); - background-color: var(--md-secondary-fg-color); - transition: background-color 250ms; -} - -.wrapper { - display: grid; - grid-template-columns: repeat(4, 1fr); - gap: 10px; - grid-auto-rows: minmax(100px, auto); -} - -.wrapper * { - border: 2px solid green; - text-align: center; - padding: 70px 0; -} - -.one { - grid-column: 1 / 2; - grid-row: 1; -} -.two { - grid-column: 2 / 3; - grid-row: 1; -} -.three { - grid-column: 3 / 4; - grid-row: 1; -} -.four { - grid-column: 4 / 5; - grid-row: 1; -} -.five { - grid-column: 1 / 3; - grid-row: 2; -} -.six { - grid-column: 3 / 5; - grid-row: 2; -} - -/* Jupyter Stuff */ -.jupyter-wrapper .jp-CodeCell .jp-Cell-inputWrapper .jp-InputPrompt { - display: none !important; -} diff --git a/hsfs/docs/css/dropdown.css b/hsfs/docs/css/dropdown.css deleted file mode 100644 index 531f7b10d..000000000 --- a/hsfs/docs/css/dropdown.css +++ /dev/null @@ -1,55 +0,0 @@ -/* Style The Dropdown Button */ -.dropbtn { - color: white; - border: none; - cursor: pointer; -} - -.md-tabs__list { - contain: inherit; -} - -.md-tabs { - overflow: inherit; -} - - -/* The container

- needed to position the dropdown content */ -.dropdown { - position: absolute; - display: inline-block; -} - -/* Dropdown Content (Hidden by Default) */ -.dropdown-content { - display: none; - font-size: 13px; - position: absolute; - background-color: #f9f9f9; - min-width: 160px; - box-shadow: 0px 8px 16px 0px rgba(0, 0, 0, 0.2); - z-index: 1000; - border-radius: 2px; - left: -15px; -} - -/* Links inside the dropdown */ -.dropdown-content a { - color: black; - padding: 12px 16px; - text-decoration: none; - display: block; -} - -/* Change color of dropdown links on hover */ -.dropdown-content a:hover { - background-color: #f1f1f1 -} - -/* Show the dropdown menu on hover */ -.dropdown:hover .dropdown-content { - display: block; -} - -/* Change the background color of the dropdown button when the dropdown content is shown */ -.dropdown:hover .dropbtn {} \ No newline at end of file diff --git a/hsfs/docs/css/marctech.css b/hsfs/docs/css/marctech.css deleted file mode 100644 index 8bb58c97b..000000000 --- a/hsfs/docs/css/marctech.css +++ /dev/null @@ -1,1047 +0,0 @@ -:root { - --md-primary-fg-color: #1EB382; - --md-secondary-fg-color: #188a64; - --md-tertiary-fg-color: #0d493550; - --md-quaternary-fg-color: #fdfdfd; - --md-fiftuary-fg-color: #2471cf; - --border-radius-variable: 5px; - --border-width:1px; - } - - .marctech_main a{ - color: var(--md-fiftuary-fg-color); - border-bottom: 1px dotted var(--md-fiftuary-fg-color) !important; - text-decoration: dotted !important;} - - .marctech_main a:hover{ - border-bottom: 1px dotted var(--md-primary-fg-color)!important; - } - - .marctech_main a:visited{ - color: var(--md-tertiary-fg-color); - border-bottom: 1px dotted var(--md-tertiary-fg-color) !important; - - } - - .w-layout-grid { - display: -ms-grid; - display: grid; - grid-auto-columns: 1fr; - -ms-grid-columns: 1fr 1fr; - grid-template-columns: 1fr 1fr; - -ms-grid-rows: auto auto; - grid-template-rows: auto auto; - grid-row-gap: 16px; - grid-column-gap: 16px; - } - - .image_logo{ - width: 69%; - background-color: white; - z-index: 50; - padding: 0px 15px 0px 15px; - margin-bottom: 10px; - } - - .layer_02{ - pointer-events: none; - } - - .round-frame{ - pointer-events: initial; - } - - .marctech_main { - margin-top:-20px; - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -webkit-flex-direction: column; - -ms-flex-direction: column; - flex-direction: column; - -webkit-box-align: center; - -webkit-align-items: center; - -ms-flex-align: center; - align-items: center; - margin-bottom: 55px; - } - - .collumns { - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - height: 100%; - -webkit-box-align: stretch; - -webkit-align-items: stretch; - -ms-flex-align: stretch; - align-items: stretch; - } - - .col_heading { - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - -webkit-box-align: center; - -webkit-align-items: center; - -ms-flex-align: center; - align-items: center; - } - - .enterprisefs { - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -webkit-flex-direction: column; - -ms-flex-direction: column; - flex-direction: column; - -webkit-box-align: center; - -webkit-align-items: center; - -ms-flex-align: center; - align-items: center; - } - - .enterprise_ai { - -webkit-align-self: center; - -ms-flex-item-align: center; - -ms-grid-row-align: center; - align-self: center; - -webkit-box-flex: 1; - -webkit-flex: 1; - -ms-flex: 1; - flex: 1; - } - - .side-content { - z-index: 0; - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - width: 240px; - height: 100%; - margin-top: 10px; - margin-bottom: 10px; - padding: 20px 10px; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -webkit-flex-direction: column; - -ms-flex-direction: column; - flex-direction: column; - -webkit-box-pack: center; - -webkit-justify-content: center; - -ms-flex-pack: center; - justify-content: center; - -webkit-box-align: center; - -webkit-align-items: center; - -ms-flex-align: center; - align-items: center; - -webkit-align-content: flex-start; - -ms-flex-line-pack: start; - align-content: flex-start; - border-style: solid; - border-width: var(--border-width); - border-color: #585858; - border-radius: 10px; - background-color:var(--md-quaternary-fg-color); - } - .body { - padding: 40px; - font-family: Roboto, sans-serif; - } - - .green { - color: #1eb182; - font-size: 1.2vw; - } - - .rec_frame { - position: relative; - z-index: 1; - display: inline-block; - min-width: 150px; - margin-top: 10px; - margin-right: 10px; - margin-left: 10px; - padding: 10px 10px; - border-style: solid; - border-width: var(--border-width); - border-color: #585858; - border-radius: 10px; - background-color: #fff; - box-shadow: 4px 4px 0 0 rgba(88, 88, 88, 0.16); - -webkit-transition: box-shadow 200ms ease, border-color 200ms ease; - transition: box-shadow 200ms ease, border-color 200ms ease; - color: #585858; - text-align: center; - cursor: pointer; - } - - .rec_frame:hover { - border-color: #c2c2c2; - box-shadow: none; - } - - .name_item { - font-size: 0.7rem; - line-height: 120%; - font-weight: 700; - } - - .name_item.db { - position: relative; - z-index: 3; - text-align: left; - } - - .name_item.small { - font-size: 0.6rem; - font-weight: 500; - } - - .name_item.ingrey { - padding-bottom: 20px; - } - - .db_frame-mid { - position: relative; - z-index: 1; - margin-top: -8px; - padding: 5px 2px; - border-style: solid; - border-width: var(--border-width); - border-color: #585858; - border-radius: 0px 0% 50% 50%; - background-color: #fff; - color: #585858; - text-align: center; - } - - .db_frame-top { - position: relative; - z-index: 2; - padding: 5px 2px; - border-style: solid; - border-width: var(--border-width); - border-color: #585858; - border-radius: 50%; - background-color: #fff; - color: #585858; - text-align: center; - } - - .icondb { - position: relative; - width: 25px; - min-width: 25px; - margin-right: 10px; - } - - .db_frame { - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - width: 150px; - height: 55px; - padding: 20px 10px; - -webkit-box-align: center; - -webkit-align-items: center; - -ms-flex-align: center; - align-items: center; - border-style: solid; - border-width: var(--border-width); - border-color: #585858; - border-radius: 10px; - background-color: #fff; - box-shadow: 4px 4px 0 0 rgba(88, 88, 88, 0.16); - -webkit-transition: box-shadow 200ms ease, border-color 200ms ease; - transition: box-shadow 200ms ease, border-color 200ms ease; - color: #585858; - text-align: center; - cursor: pointer; - } - - .db_frame:hover { - border-color: #c2c2c2; - box-shadow: none; - } - - .grid { - -ms-grid-rows: auto auto auto; - grid-template-rows: auto auto auto; - } - - .arrowdown { - position: relative; - z-index: 0; - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - margin-top: -10px; - -webkit-box-pack: center; - -webkit-justify-content: center; - -ms-flex-pack: center; - justify-content: center; - } - - .heading_MT { - margin-top: 0px !important; - margin-bottom: 0px !important; - font-size: 1.3rem !important; - white-space: nowrap !important; - } - - .head_col { - padding-left: 10px; - } - - .MT_heading3 { - margin-top: 0px !important ; - font-size: 0.8rem !important; - } - - .MT_heading3.green { - color: #1eb182 !important; - } - - .column_sides { - position: relative; - z-index: 2; - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -webkit-flex-direction: column; - -ms-flex-direction: column; - flex-direction: column; - -webkit-box-pack: justify; - -webkit-justify-content: space-between; - -ms-flex-pack: justify; - justify-content: space-between; - -webkit-box-align: center; - -webkit-align-items: center; - -ms-flex-align: center; - align-items: center; - } - - .hopsicon { - width: 45px; - height: 45px; - } - - .column_center { - z-index: 10; - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -webkit-flex-direction: column; - -ms-flex-direction: column; - flex-direction: column; - -webkit-box-pack: center; - -webkit-justify-content: center; - -ms-flex-pack: center; - justify-content: center; - -webkit-box-align: center; - -webkit-align-items: center; - -ms-flex-align: center; - align-items: center; - } - - .center-content { - z-index: -50; - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - width: 750px; - height: 670px; - margin-top: 10px; - margin-bottom: 10px; - padding: 20px 10px; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -webkit-flex-direction: column; - -ms-flex-direction: column; - flex-direction: column; - -webkit-box-pack: center; - -webkit-justify-content: center; - -ms-flex-pack: center; - justify-content: center; - -webkit-box-align: center; - -webkit-align-items: center; - -ms-flex-align: center; - align-items: center; - -webkit-align-content: center; - -ms-flex-line-pack: center; - align-content: center; - border-radius: 10px; - background-color: transparent; - } - - .image { - width: 260px; - } - - .layer_01 { - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -webkit-flex-direction: column; - -ms-flex-direction: column; - flex-direction: column; - -webkit-box-pack: center; - -webkit-justify-content: center; - -ms-flex-pack: center; - justify-content: center; - -webkit-box-align: stretch; - -webkit-align-items: stretch; - -ms-flex-align: stretch; - align-items: stretch; - } - - .name_center { - font-size: 1rem; - font-weight: 700; - } - - .rec_frame_main { - position: relative; - z-index: 1; - margin-top: 10px; - margin-right: 10px; - margin-left: 10px; - padding: 5px 10px; - border-style: solid; - border-width: var(--border-width); - border-color: #1eb182; - border-radius: 10px; - background-color: #e6fdf6; - box-shadow: 4px 4px 0 0 #dcf7ee; - -webkit-transition: box-shadow 200ms ease, border-color 200ms ease; - transition: box-shadow 200ms ease, border-color 200ms ease; - color: #1eb182; - text-align: center; - cursor: pointer; - } - - .rec_frame_main:hover { - border-color: #9fecd4; - box-shadow: none; - } - - .rec_frame_main.no_content { - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - height: 100%; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -webkit-flex-direction: column; - -ms-flex-direction: column; - flex-direction: column; - -webkit-box-pack: center; - -webkit-justify-content: center; - -ms-flex-pack: center; - justify-content: center; - -webkit-box-align: center; - -webkit-align-items: center; - -ms-flex-align: center; - align-items: center; - box-shadow: 4px 4px 0 0 #dcf7ee; - } - - .rec_frame_main.no_content:hover { - border-color: #1eb182; - box-shadow: 4px 4px 0 0 rgba(88, 88, 88, 0.16); - } - - .name_item_02 { - font-size: 0.85rem; - font-weight: 700; - } - - .grid-infra { - padding-top: 20px; - -ms-grid-columns: 1fr 1fr 1fr 1fr; - grid-template-columns: 1fr 1fr 1fr 1fr; - -ms-grid-rows: auto; - grid-template-rows: auto; - } - - .rec_frame_main-white { - position: relative; - z-index: 1; - display: inline-block; - width: 100%; - margin-top: 10px; - margin-bottom: 10px; - padding: 5px 10px; - border-style: solid; - border-width: var(--border-width); - border-color: #1eb182; - border-radius: 10px; - background-color: #fff; - box-shadow: 4px 4px 0 0 rgba(88, 88, 88, 0.16); - -webkit-transition: box-shadow 200ms ease, border-color 200ms ease; - transition: box-shadow 200ms ease, border-color 200ms ease; - color: #1eb182; - text-align: center; - cursor: pointer; - } - - .rec_frame_main-white:hover { - border-color: #c2c2c2; - box-shadow: none; - } - - .rec_frame_main-white.dotted { - border-style: dotted; - } - - .column { - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -webkit-flex-direction: column; - -ms-flex-direction: column; - flex-direction: column; - -webkit-box-pack: justify; - -webkit-justify-content: space-between; - -ms-flex-pack: justify; - justify-content: space-between; - -webkit-box-align: stretch; - -webkit-align-items: stretch; - -ms-flex-align: stretch; - align-items: stretch; - } - - .columns_center { - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - -webkit-box-orient: horizontal; - -webkit-box-direction: normal; - -webkit-flex-direction: row; - -ms-flex-direction: row; - flex-direction: row; - -webkit-box-pack: justify; - -webkit-justify-content: space-between; - -ms-flex-pack: justify; - justify-content: space-between; - } - - .non-bold { - font-weight: 400; - } - - .logo-holder { - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - -webkit-box-pack: center; - -webkit-justify-content: center; - -ms-flex-pack: center; - justify-content: center; - } - - .infra { - text-align: center; - position: relative; - z-index: 30; - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - padding: 10px; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -webkit-flex-direction: column; - -ms-flex-direction: column; - flex-direction: column; - -webkit-box-align: center; - -webkit-align-items: center; - -ms-flex-align: center; - align-items: center; - border: 1px dashed #000; - border-radius: 6px; - background-color: #fff; - cursor: pointer; - } - - .infra:hover { - border-style: solid; - border-color: #585858; - } - - .text_and_icon { - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - -webkit-box-pack: center; - -webkit-justify-content: center; - -ms-flex-pack: center; - justify-content: center; - -webkit-box-align: center; - -webkit-align-items: center; - -ms-flex-align: center; - align-items: center; - } - - .svg_icon { - width: 33px; - margin-right: 10px; - margin-left: 10px; - } - - .layer_02 { - position: absolute; - z-index: 10; - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - width: 96%; - height: 90%; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -webkit-flex-direction: column; - -ms-flex-direction: column; - flex-direction: column; - -webkit-box-pack: center; - -webkit-justify-content: center; - -ms-flex-pack: center; - justify-content: center; - -webkit-box-align: stretch; - -webkit-align-items: stretch; - -ms-flex-align: stretch; - align-items: stretch; - border-style: solid; - border-width: calc (var(--border-width)*2); - border-color: #bbbbbb50 ; - border-radius: 100%; - background-color: transparent; - } - - .round-frame { - position: absolute; - left: 0%; - top: auto; - right: auto; - bottom: 0%; - z-index: 10; - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - width: 120px; - height: 120px; - margin: 10px; - padding: 20px; - -webkit-box-pack: center; - -webkit-justify-content: center; - -ms-flex-pack: center; - justify-content: center; - -webkit-box-align: center; - -webkit-align-items: center; - -ms-flex-align: center; - align-items: center; - border-style: solid; - border-width: var(--border-width); - border-color: #585858; - border-radius: 100%; - background-color: #fff; - outline-color: #fff; - outline-offset: 0px; - outline-style: solid; - outline-width: 7px; - -webkit-transition: box-shadow 200ms ease, border-color 200ms ease; - transition: box-shadow 200ms ease, border-color 200ms ease; - color: #585858; - text-align: center; - cursor: pointer; - } - - .round-frame:hover { - border-color: #c2c2c2; - box-shadow: none; - } - - .round-frame.top-left { - left: 4%; - top: 15%; - right: auto; - bottom: auto; - } - - .round-frame.bottom-left { - left: 4%; - bottom: 15%; - } - - .round-frame.top-right { - left: auto; - top: 15%; - right: 4%; - bottom: auto; - } - - .round-frame.bottom-right { - left: auto; - top: auto; - right: 4%; - bottom: 15%; - padding: 10px; - } - - .side-holder { - z-index: -1; - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - height: 630px; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -webkit-flex-direction: column; - -ms-flex-direction: column; - flex-direction: column; - -webkit-box-pack: center; - -webkit-justify-content: center; - -ms-flex-pack: center; - justify-content: center; - } - - .infra-icon { - width: 25px; - height: 25px; - } - - .div-block { - display: -webkit-box; - display: -webkit-flex; - display: -ms-flexbox; - display: flex; - height: 100%; - -webkit-box-orient: vertical; - -webkit-box-direction: normal; - -webkit-flex-direction: column; - -ms-flex-direction: column; - flex-direction: column; - -webkit-box-pack: justify; - -webkit-justify-content: space-between; - -ms-flex-pack: justify; - justify-content: space-between; - } - - #w-node-a2a9b648-f5dd-74e5-e1c2-f7aaf4fa1fcd-46672785 { - -ms-grid-column: span 1; - grid-column-start: span 1; - -ms-grid-column-span: 1; - grid-column-end: span 1; - -ms-grid-row: span 1; - grid-row-start: span 1; - -ms-grid-row-span: 1; - grid-row-end: span 1; - } - - #w-node-_466aa2bf-88bf-5a65-eab4-fc1eb95e7384-46672785 { - -ms-grid-column: span 1; - grid-column-start: span 1; - -ms-grid-column-span: 1; - grid-column-end: span 1; - -ms-grid-row: span 1; - grid-row-start: span 1; - -ms-grid-row-span: 1; - grid-row-end: span 1; - } - - #w-node-_87009ba3-d9a6-e0b7-4cce-581190a19cf3-46672785 { - -ms-grid-column: span 1; - grid-column-start: span 1; - -ms-grid-column-span: 1; - grid-column-end: span 1; - -ms-grid-row: span 1; - grid-row-start: span 1; - -ms-grid-row-span: 1; - grid-row-end: span 1; - } - - #w-node-_4a479fbb-90c7-9f47-d439-20aa6a224339-46672785 { - -ms-grid-column: span 1; - grid-column-start: span 1; - -ms-grid-column-span: 1; - grid-column-end: span 1; - -ms-grid-row: span 1; - grid-row-start: span 1; - -ms-grid-row-span: 1; - grid-row-end: span 1; - } - - - /* - - - inherited from the original template - - */ - - .w-container .w-row { - margin-left: -10px; - margin-right: -10px; - } - .w-row:before, - .w-row:after { - content: " "; - display: table; - grid-column-start: 1; - grid-row-start: 1; - grid-column-end: 2; - grid-row-end: 2; - } - .w-row:after { - clear: both; - } - .w-row .w-row { - margin-left: 0; - margin-right: 0; - } - .w-col { - position: relative; - float: left; - width: 100%; - min-height: 1px; - padding-left: 10px; - padding-right: 10px; - } - .w-col .w-col { - padding-left: 0; - padding-right: 0; - } - .w-col-1 { - width: 8.33333333%; - } - .w-col-2 { - width: 16.66666667%; - } - .w-col-3 { - width: 25%; - } - .w-col-4 { - width: 33.33333333%; - } - .w-col-5 { - width: 41.66666667%; - } - .w-col-6 { - width: 50%; - } - .w-col-7 { - width: 58.33333333%; - } - .w-col-8 { - width: 66.66666667%; - } - .w-col-9 { - width: 75%; - } - .w-col-10 { - width: 83.33333333%; - } - .w-col-11 { - width: 91.66666667%; - } - .w-col-12 { - width: 100%; - } - .w-hidden-main { - display: none !important; - } - @media screen and (max-width: 991px) { - .w-container { - max-width: 728px; - } - .w-hidden-main { - display: inherit !important; - } - .w-hidden-medium { - display: none !important; - } - .w-col-medium-1 { - width: 8.33333333%; - } - .w-col-medium-2 { - width: 16.66666667%; - } - .w-col-medium-3 { - width: 25%; - } - .w-col-medium-4 { - width: 33.33333333%; - } - .w-col-medium-5 { - width: 41.66666667%; - } - .w-col-medium-6 { - width: 50%; - } - .w-col-medium-7 { - width: 58.33333333%; - } - .w-col-medium-8 { - width: 66.66666667%; - } - .w-col-medium-9 { - width: 75%; - } - .w-col-medium-10 { - width: 83.33333333%; - } - .w-col-medium-11 { - width: 91.66666667%; - } - .w-col-medium-12 { - width: 100%; - } - .w-col-stack { - width: 100%; - left: auto; - right: auto; - } - } - @media screen and (max-width: 767px) { - .w-hidden-main { - display: inherit !important; - } - .w-hidden-medium { - display: inherit !important; - } - .w-hidden-small { - display: none !important; - } - .w-row, - .w-container .w-row { - margin-left: 0; - margin-right: 0; - } - .w-col { - width: 100%; - left: auto; - right: auto; - } - .w-col-small-1 { - width: 8.33333333%; - } - .w-col-small-2 { - width: 16.66666667%; - } - .w-col-small-3 { - width: 25%; - } - .w-col-small-4 { - width: 33.33333333%; - } - .w-col-small-5 { - width: 41.66666667%; - } - .w-col-small-6 { - width: 50%; - } - .w-col-small-7 { - width: 58.33333333%; - } - .w-col-small-8 { - width: 66.66666667%; - } - .w-col-small-9 { - width: 75%; - } - .w-col-small-10 { - width: 83.33333333%; - } - .w-col-small-11 { - width: 91.66666667%; - } - .w-col-small-12 { - width: 100%; - } - } - @media screen and (max-width: 479px) { - .w-container { - max-width: none; - } - .w-hidden-main { - display: inherit !important; - } - .w-hidden-medium { - display: inherit !important; - } - .w-hidden-small { - display: inherit !important; - } - .w-hidden-tiny { - display: none !important; - } - .w-col { - width: 100%; - } - .w-col-tiny-1 { - width: 8.33333333%; - } - .w-col-tiny-2 { - width: 16.66666667%; - } - .w-col-tiny-3 { - width: 25%; - } - .w-col-tiny-4 { - width: 33.33333333%; - } - .w-col-tiny-5 { - width: 41.66666667%; - } - .w-col-tiny-6 { - width: 50%; - } - .w-col-tiny-7 { - width: 58.33333333%; - } - .w-col-tiny-8 { - width: 66.66666667%; - } - .w-col-tiny-9 { - width: 75%; - } - .w-col-tiny-10 { - width: 83.33333333%; - } - .w-col-tiny-11 { - width: 91.66666667%; - } - .w-col-tiny-12 { - width: 100%; - } - } diff --git a/hsfs/docs/css/version-select.css b/hsfs/docs/css/version-select.css deleted file mode 100644 index 3b908ae84..000000000 --- a/hsfs/docs/css/version-select.css +++ /dev/null @@ -1,36 +0,0 @@ -@media only screen and (max-width:76.1875em) { -} - -#version-selector select.form-control { - appearance: none; - -webkit-appearance: none; - -moz-appearance: none; - - background-color: #F5F5F5; - - background-position: center right; - background-repeat: no-repeat; - border: 0px; - border-radius: 2px; - /* box-shadow: 0px 1px 3px rgb(0 0 0 / 10%); */ - color: inherit; - width: -webkit-fill-available; - width: -moz-available; - max-width: 200px; - font-size: inherit; - /* font-weight: 600; */ - margin: 10px; - overflow: hidden; - padding: 7px 10px; - text-overflow: ellipsis; - white-space: nowrap; -} - -#version-selector::after { - content: '⌄'; - font-family: inherit; - font-size: 22px; - margin: -35px; - vertical-align: 7%; - padding-bottom: 10px; -} diff --git a/hsfs/docs/index.md b/hsfs/docs/index.md deleted file mode 100644 index a13ea2ce5..000000000 --- a/hsfs/docs/index.md +++ /dev/null @@ -1,201 +0,0 @@ -# Hopsworks Feature Store - -

- Hopsworks Community - Hopsworks Feature Store Documentation - python - PyPiStatus - Scala/Java Artifacts - Downloads - Ruff - License -

- -HSFS is the library to interact with the Hopsworks Feature Store. The library makes creating new features, feature groups and training datasets easy. - -The library is environment independent and can be used in two modes: - -- Spark mode: For data engineering jobs that create and write features into the feature store or generate training datasets. It requires a Spark environment such as the one provided in the Hopsworks platform or Databricks. In Spark mode, HSFS provides bindings both for Python and JVM languages. - -- Python mode: For data science jobs to explore the features available in the feature store, generate training datasets and feed them in a training pipeline. Python mode requires just a Python interpreter and can be used both in Hopsworks from Python Jobs/Jupyter Kernels, Amazon SageMaker or KubeFlow. - -The library automatically configures itself based on the environment it is run. -However, to connect from an external environment such as Databricks or AWS Sagemaker, -additional connection information, such as host and port, is required. For more information checkout the [Hopsworks documentation](https://docs.hopsworks.ai/latest/). - -## Getting Started On Hopsworks - -Get started easily by registering an account on [Hopsworks Serverless](https://app.hopsworks.ai/). Create your project and a [new Api key](https://docs.hopsworks.ai/latest/user_guides/projects/api_key/create_api_key/). In a new python environment with Python 3.8 or higher, install the [client library](https://docs.hopsworks.ai/latest/user_guides/client_installation/) using pip: - -```bash -# Get all Hopsworks SDKs: Feature Store, Model Serving and Platform SDK -pip install hopsworks -# or minimum install with the Feature Store SDK -pip install hsfs[python] -# if using zsh don't forget the quotes -pip install 'hsfs[python]' -``` - -You can start a notebook and instantiate a connection and get the project feature store handler. - -```python -import hopsworks - -project = hopsworks.login() # you will be prompted for your api key -fs = project.get_feature_store() -``` - -or using `hsfs` directly: - -```python -import hsfs - -connection = hsfs.connection( - host="c.app.hopsworks.ai", # - project="your-project", - api_key_value="your-api-key", -) -fs = connection.get_feature_store() -``` - -Create a new feature group to start inserting feature values. -```python -fg = fs.create_feature_group("rain", - version=1, - description="Rain features", - primary_key=['date', 'location_id'], - online_enabled=True) - -fg.save(dataframe) -``` - -Upsert new data in to the feature group with `time_travel_format="HUDI"`". -```python -fg.insert(upsert_df) -``` - -Retrieve commit timeline metdata of the feature group with `time_travel_format="HUDI"`". -```python -fg.commit_details() -``` - -"Reading feature group as of specific point in time". -```python -fg = fs.get_feature_group("rain", 1) -fg.read("2020-10-20 07:34:11").show() -``` - -Read updates that occurred between specified points in time. -```python -fg = fs.get_feature_group("rain", 1) -fg.read_changes("2020-10-20 07:31:38", "2020-10-20 07:34:11").show() -``` - -Join features together -```python -feature_join = rain_fg.select_all() - .join(temperature_fg.select_all(), on=["date", "location_id"]) - .join(location_fg.select_all()) -feature_join.show(5) -``` - -join feature groups that correspond to specific point in time -```python -feature_join = rain_fg.select_all() - .join(temperature_fg.select_all(), on=["date", "location_id"]) - .join(location_fg.select_all()) - .as_of("2020-10-31") -feature_join.show(5) -``` - -join feature groups that correspond to different time -```python -rain_fg_q = rain_fg.select_all().as_of("2020-10-20 07:41:43") -temperature_fg_q = temperature_fg.select_all().as_of("2020-10-20 07:32:33") -location_fg_q = location_fg.select_all().as_of("2020-10-20 07:33:08") -joined_features_q = rain_fg_q.join(temperature_fg_q).join(location_fg_q) -``` - -Use the query object to create a training dataset: -```python -td = fs.create_training_dataset("rain_dataset", - version=1, - data_format="tfrecords", - description="A test training dataset saved in TfRecords format", - splits={'train': 0.7, 'test': 0.2, 'validate': 0.1}) - -td.save(feature_join) -``` - -A short introduction to the Scala API: -```scala -import com.logicalclocks.hsfs._ -val connection = HopsworksConnection.builder().build() -val fs = connection.getFeatureStore(); -val attendances_features_fg = fs.getFeatureGroup("games_features", 1); -attendances_features_fg.show(1) -``` - -You can find more examples on how to use the library in our [hops-examples](https://github.com/logicalclocks/hops-examples) repository. - -## Usage - -Usage data is collected for improving quality of the library. It is turned on by default if the backend -is "c.app.hopsworks.ai". To turn it off, use one of the following way: -```python -# use environment variable -import os -os.environ["ENABLE_HOPSWORKS_USAGE"] = "false" - -# use `disable_usage_logging` -import hsfs -hsfs.disable_usage_logging() -``` - -The source code can be found in python/hsfs/usage.py. - -## Documentation - -Documentation is available at [Hopsworks Feature Store Documentation](https://docs.hopsworks.ai/). - -## Issues - -For general questions about the usage of Hopsworks and the Feature Store please open a topic on [Hopsworks Community](https://community.hopsworks.ai/). - -Please report any issue using [Github issue tracking](https://github.com/logicalclocks/feature-store-api/issues). - -Please attach the client environment from the output below in the issue: -```python -import hopsworks -import hsfs -hopsworks.login().get_feature_store() -print(hsfs.get_env()) -``` - -## Contributing - -If you would like to contribute to this library, please see the [Contribution Guidelines](CONTRIBUTING.md). diff --git a/hsfs/docs/js/dropdown.js b/hsfs/docs/js/dropdown.js deleted file mode 100644 index 2618e0ce7..000000000 --- a/hsfs/docs/js/dropdown.js +++ /dev/null @@ -1,2 +0,0 @@ -document.getElementsByClassName("md-tabs__link")[7].style.display = "none"; -document.getElementsByClassName("md-tabs__link")[9].style.display = "none"; diff --git a/hsfs/docs/js/inject-api-links.js b/hsfs/docs/js/inject-api-links.js deleted file mode 100644 index aa5852283..000000000 --- a/hsfs/docs/js/inject-api-links.js +++ /dev/null @@ -1,32 +0,0 @@ -window.addEventListener("DOMContentLoaded", function () { - var windowPathNameSplits = window.location.pathname.split("/"); - var majorVersionRegex = new RegExp("(\\d+[.]\\d+)") - var latestRegex = new RegExp("latest"); - if (majorVersionRegex.test(windowPathNameSplits[1])) { // On landing page docs.hopsworks.api/3.0 - URL contains major version - // Version API dropdown - document.getElementById("hopsworks_api_link").href = "https://docs.hopsworks.ai/hopsworks-api/" + windowPathNameSplits[1] + "/generated/api/login/"; - document.getElementById("hsfs_api_link").href = "https://docs.hopsworks.ai/feature-store-api/" + windowPathNameSplits[1] + "/generated/api/connection_api/"; - document.getElementById("hsml_api_link").href = "https://docs.hopsworks.ai/machine-learning-api/" + windowPathNameSplits[1] + "/generated/connection_api/"; - } else { // on docs.hopsworks.api/feature-store-api/3.0 / docs.hopsworks.api/hopsworks-api/3.0 / docs.hopsworks.api/machine-learning-api/3.0 - if (latestRegex.test(windowPathNameSplits[2]) || latestRegex.test(windowPathNameSplits[1])) { - var majorVersion = "latest"; - } else { - - var apiVersion = windowPathNameSplits[2]; - var majorVersion = apiVersion.match(majorVersionRegex)[0]; - } - // Version main navigation - document.getElementsByClassName("md-tabs__link")[0].href = "https://docs.hopsworks.ai/" + majorVersion; - document.getElementsByClassName("md-tabs__link")[1].href = "https://colab.research.google.com/github/logicalclocks/hopsworks-tutorials/blob/master/quickstart.ipynb"; - document.getElementsByClassName("md-tabs__link")[2].href = "https://docs.hopsworks.ai/" + majorVersion + "/tutorials/"; - document.getElementsByClassName("md-tabs__link")[3].href = "https://docs.hopsworks.ai/" + majorVersion + "/concepts/hopsworks/"; - document.getElementsByClassName("md-tabs__link")[4].href = "https://docs.hopsworks.ai/" + majorVersion + "/user_guides/"; - document.getElementsByClassName("md-tabs__link")[5].href = "https://docs.hopsworks.ai/" + majorVersion + "/setup_installation/aws/getting_started/"; - document.getElementsByClassName("md-tabs__link")[6].href = "https://docs.hopsworks.ai/" + majorVersion + "/admin/"; - // Version API dropdown - document.getElementById("hopsworks_api_link").href = "https://docs.hopsworks.ai/hopsworks-api/" + majorVersion + "/generated/api/login/"; - document.getElementById("hsfs_api_link").href = "https://docs.hopsworks.ai/feature-store-api/" + majorVersion + "/generated/api/connection_api/"; - document.getElementById("hsfs_javadoc_link").href = "https://docs.hopsworks.ai/feature-store-api/" + majorVersion + "/javadoc"; - document.getElementById("hsml_api_link").href = "https://docs.hopsworks.ai/machine-learning-api/" + majorVersion + "/generated/connection_api/"; - } -}); diff --git a/hsfs/docs/js/version-select.js b/hsfs/docs/js/version-select.js deleted file mode 100644 index fcac029e3..000000000 --- a/hsfs/docs/js/version-select.js +++ /dev/null @@ -1,64 +0,0 @@ -window.addEventListener("DOMContentLoaded", function() { - // This is a bit hacky. Figure out the base URL from a known CSS file the - // template refers to... - var ex = new RegExp("/?css/version-select.css$"); - var sheet = document.querySelector('link[href$="version-select.css"]'); - - var ABS_BASE_URL = sheet.href.replace(ex, ""); - var CURRENT_VERSION = ABS_BASE_URL.split("/").pop(); - - function makeSelect(options, selected) { - var select = document.createElement("select"); - select.classList.add("form-control"); - - options.forEach(function(i) { - var option = new Option(i.text, i.value, undefined, - i.value === selected); - select.add(option); - }); - - return select; - } - - var xhr = new XMLHttpRequest(); - xhr.open("GET", ABS_BASE_URL + "/../versions.json"); - xhr.onload = function() { - var versions = JSON.parse(this.responseText); - - var realVersion = versions.find(function(i) { - return i.version === CURRENT_VERSION || - i.aliases.includes(CURRENT_VERSION); - }).version; - var latestVersion = versions.find(function(i) { - return i.aliases.includes("latest"); - }).version; - let outdated_banner = document.querySelector('div[data-md-color-scheme="default"][data-md-component="outdated"]'); - if (realVersion !== latestVersion) { - outdated_banner.removeAttribute("hidden"); - } else { - outdated_banner.setAttribute("hidden", ""); - } - - var select = makeSelect(versions.map(function(i) { - var allowedAliases = ["dev", "latest"] - if (i.aliases.length > 0) { - var aliasString = " [" + i.aliases.filter(function (str) { return allowedAliases.includes(str); }).join(", ") + "]"; - } else { - var aliasString = ""; - } - return {text: i.title + aliasString, value: i.version}; - }), realVersion); - select.addEventListener("change", function(event) { - window.location.href = ABS_BASE_URL + "/../" + this.value + "/generated/api/connection_api/"; - }); - - var container = document.createElement("div"); - container.id = "version-selector"; - // container.className = "md-nav__item"; - container.appendChild(select); - - var sidebar = document.querySelector(".md-nav--primary > .md-nav__list"); - sidebar.parentNode.insertBefore(container, sidebar.nextSibling); - }; - xhr.send(); -}); diff --git a/hsfs/docs/overrides/main.html b/hsfs/docs/overrides/main.html deleted file mode 100644 index ecb09de07..000000000 --- a/hsfs/docs/overrides/main.html +++ /dev/null @@ -1,8 +0,0 @@ -{% extends "base.html" %} - -{% block outdated %} -You're not viewing the latest version of the documentation. - - Click here to go to latest. - -{% endblock %} diff --git a/hsfs/docs/templates/api/connection_api.md b/hsfs/docs/templates/api/connection_api.md deleted file mode 100644 index 19e13f3eb..000000000 --- a/hsfs/docs/templates/api/connection_api.md +++ /dev/null @@ -1,11 +0,0 @@ -# Connection - -{{connection}} - -## Properties - -{{connection_properties}} - -## Methods - -{{connection_methods}} diff --git a/hsfs/docs/templates/api/embedding_feature_api.md b/hsfs/docs/templates/api/embedding_feature_api.md deleted file mode 100644 index c054672d0..000000000 --- a/hsfs/docs/templates/api/embedding_feature_api.md +++ /dev/null @@ -1,7 +0,0 @@ -# EmbeddingFeature - -{{embedding_feature}} - -## Properties - -{{embedding_feature_properties}} diff --git a/hsfs/docs/templates/api/embedding_index_api.md b/hsfs/docs/templates/api/embedding_index_api.md deleted file mode 100644 index d336e0ddb..000000000 --- a/hsfs/docs/templates/api/embedding_index_api.md +++ /dev/null @@ -1,12 +0,0 @@ -# EmbeddingIndex - -{{embedding_index}} - -## Properties - -{{embedding_index_properties}} - -## Methods - -{{embedding_index_methods}} - diff --git a/hsfs/docs/templates/api/expectation_api.md b/hsfs/docs/templates/api/expectation_api.md deleted file mode 100644 index 7ba4110c1..000000000 --- a/hsfs/docs/templates/api/expectation_api.md +++ /dev/null @@ -1,20 +0,0 @@ -# Expectation - -{{expectation}} - -## Properties - -{{expectation_properties}} - -## Methods - -{{expectation_methods}} - -## Creation -{{expectation_create}} - -## Retrieval - -{{expectation_getall}} - -{{expectation_get}} diff --git a/hsfs/docs/templates/api/expectation_suite_api.md b/hsfs/docs/templates/api/expectation_suite_api.md deleted file mode 100644 index a07ac5f8a..000000000 --- a/hsfs/docs/templates/api/expectation_suite_api.md +++ /dev/null @@ -1,41 +0,0 @@ -# Expectation Suite - -{{expectation_suite}} - -## Creation with Great Expectations - -```python3 -import great_expectations as ge - -expectation_suite = ge.core.ExpectationSuite( - "new_expectation_suite", - expectations=[ - ge.core.ExpectationConfiguration( - expectation_type="expect_column_max_to_be_between", - kwargs={ - "column": "feature", - "min_value": -1, - "max_value": 1 - } - ) - ] -) -``` - -## Attach to Feature Group - -{{expectation_suite_attach}} - -## Single Expectation API - -An API to edit the expectation list based on Great Expectations API. - -{{single_expectation_api}} - -## Properties - -{{expectation_suite_properties}} - -## Methods - -{{expectation_suite_methods}} diff --git a/hsfs/docs/templates/api/external_feature_group_api.md b/hsfs/docs/templates/api/external_feature_group_api.md deleted file mode 100644 index a982a39e8..000000000 --- a/hsfs/docs/templates/api/external_feature_group_api.md +++ /dev/null @@ -1,19 +0,0 @@ -# ExternalFeatureGroup - -{{fg}} - -## Creation - -{{fg_create}} - -## Retrieval - -{{fg_get}} - -## Properties - -{{fg_properties}} - -## Methods - -{{fg_methods}} diff --git a/hsfs/docs/templates/api/feature_api.md b/hsfs/docs/templates/api/feature_api.md deleted file mode 100644 index 8dca5ef54..000000000 --- a/hsfs/docs/templates/api/feature_api.md +++ /dev/null @@ -1,11 +0,0 @@ -# Feature - -{{feature}} - -## Properties - -{{feature_properties}} - -## Methods - -{{feature_methods}} diff --git a/hsfs/docs/templates/api/feature_descriptive_statistics_api.md b/hsfs/docs/templates/api/feature_descriptive_statistics_api.md deleted file mode 100644 index 3be8cccd3..000000000 --- a/hsfs/docs/templates/api/feature_descriptive_statistics_api.md +++ /dev/null @@ -1,7 +0,0 @@ -# Feature Descriptive Statistics - -{{feature_descriptive_statistics}} - -## Properties - -{{feature_descriptive_statistics_properties}} diff --git a/hsfs/docs/templates/api/feature_group_api.md b/hsfs/docs/templates/api/feature_group_api.md deleted file mode 100644 index 372865c4b..000000000 --- a/hsfs/docs/templates/api/feature_group_api.md +++ /dev/null @@ -1,19 +0,0 @@ -# FeatureGroup - -{{fg}} - -## Creation - -{{fg_create}} - -## Retrieval - -{{fg_get}} - -## Properties - -{{fg_properties}} - -## Methods - -{{fg_methods}} diff --git a/hsfs/docs/templates/api/feature_monitoring_config_api.md b/hsfs/docs/templates/api/feature_monitoring_config_api.md deleted file mode 100644 index 7ca9b46ff..000000000 --- a/hsfs/docs/templates/api/feature_monitoring_config_api.md +++ /dev/null @@ -1,27 +0,0 @@ -# Feature Monitoring Configuration - -{{feature_monitoring_config}} - -## Creation from Feature Group - -{{feature_monitoring_config_creation_fg}} - -## Creation from Feature View - -{{feature_monitoring_config_creation_fv}} - -## Retrieval from Feature Group - -{{feature_monitoring_config_retrieval_fg}} - -## Retrieval from Feature View - -{{feature_monitoring_config_retrieval_fv}} - -## Properties - -{{feature_monitoring_config_properties}} - -## Methods - -{{feature_monitoring_config_methods}} diff --git a/hsfs/docs/templates/api/feature_monitoring_result_api.md b/hsfs/docs/templates/api/feature_monitoring_result_api.md deleted file mode 100644 index 5bfca1165..000000000 --- a/hsfs/docs/templates/api/feature_monitoring_result_api.md +++ /dev/null @@ -1,11 +0,0 @@ -# Feature Monitoring Result - -{{feature_monitoring_result}} - -## Retrieval - -{{feature_monitoring_result_retrieval}} - -## Properties - -{{feature_monitoring_result_properties}} diff --git a/hsfs/docs/templates/api/feature_monitoring_window_config_api.md b/hsfs/docs/templates/api/feature_monitoring_window_config_api.md deleted file mode 100644 index 53ef23ea2..000000000 --- a/hsfs/docs/templates/api/feature_monitoring_window_config_api.md +++ /dev/null @@ -1,7 +0,0 @@ -# Feature Monitoring Window Configuration - -{{feature_monitoring_window_config}} - -## Properties - -{{feature_monitoring_window_config_properties}} diff --git a/hsfs/docs/templates/api/feature_store_api.md b/hsfs/docs/templates/api/feature_store_api.md deleted file mode 100644 index f859336f6..000000000 --- a/hsfs/docs/templates/api/feature_store_api.md +++ /dev/null @@ -1,15 +0,0 @@ -# Feature Store - -{{fs}} - -## Retrieval - -{{fs_get}} - -## Properties - -{{fs_properties}} - -## Methods - -{{fs_methods}} diff --git a/hsfs/docs/templates/api/feature_view_api.md b/hsfs/docs/templates/api/feature_view_api.md deleted file mode 100644 index c0f7df954..000000000 --- a/hsfs/docs/templates/api/feature_view_api.md +++ /dev/null @@ -1,21 +0,0 @@ -# Feature View - -{{fv}} - -## Creation - -{{fv_create}} - -## Retrieval - -{{fv_get}} - -{{fvs_get}} - -## Properties - -{{fv_properties}} - -## Methods - -{{fv_methods}} diff --git a/hsfs/docs/templates/api/job.md b/hsfs/docs/templates/api/job.md deleted file mode 100644 index 9ad68d976..000000000 --- a/hsfs/docs/templates/api/job.md +++ /dev/null @@ -1,11 +0,0 @@ -# Job - -{{job}} - -## Methods - -{{job_methods}} - -## Job Configuration - -{{job_configuration}} diff --git a/hsfs/docs/templates/api/links.md b/hsfs/docs/templates/api/links.md deleted file mode 100644 index 62cdc7001..000000000 --- a/hsfs/docs/templates/api/links.md +++ /dev/null @@ -1,14 +0,0 @@ -# Provenance Links - -Provenance Links are objects returned by methods such as [get_feature_groups_provenance](../storage_connector_api/#get_feature_groups_provenance), [get_storage_connector_provenance](../feature_group_api/#get_storage_connector_provenance), [get_parent_feature_group](../feature_group_api/#get_parent_feature_groups), [get_generated_feature_groups](../feature_group_api/#get_generated_feature_groups), [get_generated_feature_views](../feature_group_api/#get_generated_feature_views) [get_models_provenance](../feature_view_api/#get_models_provenance) and represent sections of the provenance graph, depending on the method invoked. - -## Properties - -{{links_properties}} - -# Artifact - -Artifacts objects are part of the provenance graph and contain a minimal set of information regarding the entities (feature groups, feature views) they represent. -The provenance graph contains Artifact objects when the underlying entities have been deleted or they are corrupted or they are not accessible by the user. - -{{artifact_properties}} diff --git a/hsfs/docs/templates/api/query_api.md b/hsfs/docs/templates/api/query_api.md deleted file mode 100644 index 7cc664d96..000000000 --- a/hsfs/docs/templates/api/query_api.md +++ /dev/null @@ -1,13 +0,0 @@ -# Query - -Query objects are strictly generated by HSFS APIs called on [Feature Group objects](feature_group_api.md). -Users will never construct a Query object using the constructor of the class. -For this reason we do not provide the full documentation of the class here. - -## Methods - -{{query_methods}} - -## Properties - -{{query_properties}} diff --git a/hsfs/docs/templates/api/rule_api.md b/hsfs/docs/templates/api/rule_api.md deleted file mode 100644 index 0801e3954..000000000 --- a/hsfs/docs/templates/api/rule_api.md +++ /dev/null @@ -1,7 +0,0 @@ -# Rule - -{{rule}} - -## Properties - -{{rule_properties}} diff --git a/hsfs/docs/templates/api/rule_definition_api.md b/hsfs/docs/templates/api/rule_definition_api.md deleted file mode 100644 index 326b66db0..000000000 --- a/hsfs/docs/templates/api/rule_definition_api.md +++ /dev/null @@ -1,13 +0,0 @@ -# Rule Definition - -{{ruledefinition}} - -## Properties - -{{ruledefinition_properties}} - -## Retrieval - -{{ruledefinition_getall}} - -{{ruledefinition_get}} diff --git a/hsfs/docs/templates/api/similarity_function_type_api.md b/hsfs/docs/templates/api/similarity_function_type_api.md deleted file mode 100644 index bdfbc51c2..000000000 --- a/hsfs/docs/templates/api/similarity_function_type_api.md +++ /dev/null @@ -1,3 +0,0 @@ -# SimilarityFunctionType - -{{similarity_function_type}} diff --git a/hsfs/docs/templates/api/spine_group_api.md b/hsfs/docs/templates/api/spine_group_api.md deleted file mode 100644 index a2bdf119c..000000000 --- a/hsfs/docs/templates/api/spine_group_api.md +++ /dev/null @@ -1,19 +0,0 @@ -# SpineGroup - -{{fg}} - -## Creation - -{{fg_create}} - -## Retrieval - -{{fg_get}} - -## Properties - -{{fg_properties}} - -## Methods - -{{fg_methods}} diff --git a/hsfs/docs/templates/api/split_statistics_api.md b/hsfs/docs/templates/api/split_statistics_api.md deleted file mode 100644 index 09053ac5d..000000000 --- a/hsfs/docs/templates/api/split_statistics_api.md +++ /dev/null @@ -1,7 +0,0 @@ -# Split Statistics - -{{split_statistics}} - -## Properties - -{{split_statistics_properties}} diff --git a/hsfs/docs/templates/api/statistics_api.md b/hsfs/docs/templates/api/statistics_api.md deleted file mode 100644 index 27ed90c9d..000000000 --- a/hsfs/docs/templates/api/statistics_api.md +++ /dev/null @@ -1,7 +0,0 @@ -# Statistics - -{{statistics}} - -## Properties - -{{statistics_properties}} diff --git a/hsfs/docs/templates/api/statistics_config_api.md b/hsfs/docs/templates/api/statistics_config_api.md deleted file mode 100644 index a907d1d32..000000000 --- a/hsfs/docs/templates/api/statistics_config_api.md +++ /dev/null @@ -1,7 +0,0 @@ -# StatisticsConfig - -{{statistics_config}} - -## Properties - -{{statistics_config_properties}} diff --git a/hsfs/docs/templates/api/storage_connector_api.md b/hsfs/docs/templates/api/storage_connector_api.md deleted file mode 100644 index 1b390e72a..000000000 --- a/hsfs/docs/templates/api/storage_connector_api.md +++ /dev/null @@ -1,119 +0,0 @@ -# Storage Connector - -## Retrieval - -{{sc_get}} - -## HopsFS - -### Properties - -{{hopsfs_properties}} - -### Methods - -{{hopsfs_methods}} - -## JDBC - -### Properties - -{{jdbc_properties}} - -### Methods - -{{jdbc_methods}} - -## S3 - -### Properties - -{{s3_properties}} - -### Methods - -{{s3_methods}} - -## Redshift - -### Properties - -{{redshift_properties}} - -### Methods - -{{redshift_methods}} - -## Azure Data Lake Storage - -### Properties - -{{adls_properties}} - -### Methods - -{{adls_methods}} - -## Snowflake - -### Properties - -{{snowflake_properties}} - -### Methods - -{{snowflake_methods}} - -## Google Cloud Storage -This storage connector provides integration to Google Cloud Storage (GCS). -Once you create a connector in FeatureStore, you can transact data from a GCS bucket into a spark dataframe -by calling the `read` API. - -Authentication to GCP is handled by uploading the `JSON keyfile for service account` to the Hopsworks Project. For more information -on service accounts and creating keyfile in GCP, read [Google Cloud documentation.](https://cloud.google.com/docs/authentication/production#create_service_account -'creating service account keyfile') - -The connector also supports the optional encryption method `Customer Supplied Encryption Key` by Google. -The encryption details are stored as `Secrets` in the FeatureStore for keeping it secure. -Read more about encryption on [Google Documentation.](https://cloud.google.com/storage/docs/encryption#customer-supplied_encryption_keys) - -The storage connector uses the Google `gcs-connector-hadoop` behind the scenes. For more information, check out [Google Cloud Storage Connector for Spark and Hadoop]( -https://github.com/GoogleCloudDataproc/hadoop-connectors/tree/master/gcs#google-cloud-storage-connector-for-spark-and-hadoop 'google-cloud-storage-connector-for-spark-and-hadoop') - -### Properties - -{{gcs_properties}} - -### Methods - -{{gcs_methods}} - -## BigQuery -The BigQuery storage connector provides integration to Google Cloud BigQuery. -You can use it to run bigquery on your GCP cluster and load results into spark dataframe by calling the `read` API. - -Authentication to GCP is handled by uploading the `JSON keyfile for service account` to the Hopsworks Project. For more information -on service accounts and creating keyfile in GCP, read [Google Cloud documentation.](https://cloud.google.com/docs/authentication/production#create_service_account -'creating service account keyfile') - -The storage connector uses the Google `spark-bigquery-connector` behind the scenes. -To read more about the spark connector, like the spark options or usage, check [Apache Spark SQL connector for Google BigQuery.](https://github.com/GoogleCloudDataproc/spark-bigquery-connector#usage -'github.com/GoogleCloudDataproc/spark-bigquery-connector') - -### Properties - -{{bigquery_properties}} - -### Methods - -{{bigquery_methods}} - -## Kafka - -### Properties - -{{kafka_properties}} - -### Methods - -{{kafka_methods}} diff --git a/hsfs/docs/templates/api/training_dataset_api.md b/hsfs/docs/templates/api/training_dataset_api.md deleted file mode 100644 index a53696465..000000000 --- a/hsfs/docs/templates/api/training_dataset_api.md +++ /dev/null @@ -1,19 +0,0 @@ -# Training Dataset - -{{td}} - -## Creation - -{{td_create}} - -## Retrieval - -{{td_get}} - -## Properties - -{{td_properties}} - -## Methods - -{{td_methods}} diff --git a/hsfs/docs/templates/api/transformation_functions_api.md b/hsfs/docs/templates/api/transformation_functions_api.md deleted file mode 100644 index 249262a45..000000000 --- a/hsfs/docs/templates/api/transformation_functions_api.md +++ /dev/null @@ -1,20 +0,0 @@ -# Transformation Function - -{{transformation_function}} - -## Properties - -{{transformation_function_properties}} - -## Methods - -{{transformation_function_methods}} - -## Creation -{{create_transformation_function}} - -## Retrieval - -{{get_transformation_function}} - -{{get_transformation_functions}} diff --git a/hsfs/docs/templates/api/validation_api.md b/hsfs/docs/templates/api/validation_api.md deleted file mode 100644 index 8e1512f34..000000000 --- a/hsfs/docs/templates/api/validation_api.md +++ /dev/null @@ -1,18 +0,0 @@ -# Validation - -{{validation_result}} - -## Properties - -{{validation_result_properties}} - -## Methods - -{{expectation_methods}} - -## Validate a dataframe -{{validate}} - -## Retrieval - -{{validation_result_get}} diff --git a/hsfs/docs/templates/api/validation_report_api.md b/hsfs/docs/templates/api/validation_report_api.md deleted file mode 100644 index 435a87a03..000000000 --- a/hsfs/docs/templates/api/validation_report_api.md +++ /dev/null @@ -1,19 +0,0 @@ -# Validation Report - -{{validation_report}} - -## Creation - -{{validation_report_validate}} - -## Retrieval - -{{validation_report_get}} - -## Properties - -{{validation_report_properties}} - -## Methods - -{{validation_report_methods}} diff --git a/hsfs/java/beam/pom.xml b/hsfs/java/beam/pom.xml deleted file mode 100644 index 3b3f902ca..000000000 --- a/hsfs/java/beam/pom.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - hsfs-parent - com.logicalclocks - 4.0.0-SNAPSHOT - - 4.0.0 - - hsfs-beam - - - 2.48.0 - 3.4.0 - - - - - com.logicalclocks - hsfs - ${project.version} - compile - - - javax.xml.bind - jaxb-api - - - - - - - org.apache.beam - beam-sdks-java-core - ${beam.version} - - - - - org.apache.beam - beam-sdks-java-io-kafka - ${beam.version} - - - - - org.apache.kafka - kafka-clients - ${kafka.version} - - - - diff --git a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/FeatureStore.java b/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/FeatureStore.java deleted file mode 100644 index fd93052a3..000000000 --- a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/FeatureStore.java +++ /dev/null @@ -1,322 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.beam; - -import com.logicalclocks.hsfs.FeatureStoreBase; -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.StatisticsConfig; -import com.logicalclocks.hsfs.StorageConnector; -import com.logicalclocks.hsfs.TimeTravelFormat; -import com.logicalclocks.hsfs.TrainingDatasetBase; -import com.logicalclocks.hsfs.beam.constructor.Query; -import com.logicalclocks.hsfs.beam.engine.FeatureGroupEngine; -import com.logicalclocks.hsfs.beam.engine.FeatureViewEngine; -import com.logicalclocks.hsfs.metadata.StorageConnectorApi; -import lombok.NonNull; - -import java.io.IOException; -import java.util.List; - -public class FeatureStore extends FeatureStoreBase { - - private FeatureGroupEngine featureGroupEngine; - private FeatureViewEngine featureViewEngine; - - public FeatureStore() { - storageConnectorApi = new StorageConnectorApi(); - featureGroupEngine = new FeatureGroupEngine(); - } - - @Override - public Object createFeatureGroup() { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getFeatureGroups(@NonNull String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getOrCreateFeatureGroup(String name, Integer version) throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - - @Override - public Object getOrCreateFeatureGroup(String name, Integer version, List primaryKeys, - boolean onlineEnabled, String eventTime) throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getOrCreateFeatureGroup(String name, Integer version, List primaryKeys, - List partitionKeys, boolean onlineEnabled, String eventTime) throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getOrCreateFeatureGroup(String name, Integer version, String description, List primaryKeys, - List partitionKeys, String hudiPrecombineKey, boolean onlineEnabled, TimeTravelFormat timeTravelFormat, - StatisticsConfig statisticsConfig, String topicName, String notificationTopicName, String eventTime) - throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - /** - * Get a stream feature group object from the feature store. - * - *

Getting a stream feature group metadata handle enables to interact with the feature group, - * such as read the data or use the `Query`-API to perform joins between feature groups and create feature - * views. - * - * 

-   * {@code
-   *        // get feature store handle
-   *        FeatureStore fs = HopsworksConnection.builder().build().getFeatureStore();
-   *        FeatureGroup fg = fs.getStreamFeatureGroup("electricity_prices", 1);
-   * }
-   *
- * - * @param name the name of the feature group - * @return StreamFeatureGroup The stream feature group metadata object. - * @throws FeatureStoreException If unable to retrieve feature group from the feature store. - * @throws IOException Generic IO exception. - */ - @Override - public Object getStreamFeatureGroup(String name) throws FeatureStoreException, IOException { - LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `" - + DEFAULT_VERSION + "`."); - return getStreamFeatureGroup(name, DEFAULT_VERSION); - } - - /** - * Get a stream feature group object from the feature store. - * - *

Getting a stream feature group metadata handle enables to interact with the feature group, - * such as read the data or use the `Query`-API to perform joins between feature groups and create feature - * views. - * - * 

-   * {@code
-   *        // get feature store handle
-   *        FeatureStore fs = HopsworksConnection.builder().build().getFeatureStore();
-   *        FeatureGroup fg = fs.getStreamFeatureGroup("electricity_prices", 1);
-   * }
-   *
- * - * @param name the name of the feature group - * @param version the version of the feature group - * @return StreamFeatureGroup The stream feature group metadata object. - * @throws FeatureStoreException If unable to retrieve feature group from the feature store. - * @throws IOException Generic IO exception. - */ - @Override - public StreamFeatureGroup getStreamFeatureGroup(@NonNull String name, @NonNull Integer version) - throws FeatureStoreException, IOException { - return featureGroupEngine.getStreamFeatureGroup(this, name, version); - } - - @Override - public Object createStreamFeatureGroup() { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public StreamFeatureGroup getOrCreateStreamFeatureGroup(String name, Integer version) - throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public StreamFeatureGroup getOrCreateStreamFeatureGroup(String name, Integer version, List primaryKeys, - boolean onlineEnabled, String eventTime) throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public StreamFeatureGroup getOrCreateStreamFeatureGroup(String name, Integer version, List primaryKeys, - List partitionKeys, boolean onlineEnabled, String eventTime) throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public StreamFeatureGroup getOrCreateStreamFeatureGroup(String name, Integer version, String description, - List primaryKeys, List partitionKeys, String hudiPrecombineKey, boolean onlineEnabled, - StatisticsConfig statisticsConfig, String eventTime) throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object createExternalFeatureGroup() { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object createFeatureView() { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - /** - * Get a feature view object from the selected feature store. - * - * 
-   * {@code
-   *        // get feature store handle
-   *        FeatureStore fs = HopsworksConnection.builder().build().getFeatureStore();
-   *        FeatureView fv = fs.getFeatureView("fv_name", 1);
-   * }
-   *
- * - * @param name Name of the feature view. - * @param version Version to get. - * @return FeatureView The feature view metadata object. - * @throws FeatureStoreException If unable to retrieve FeatureView from the feature store. - * @throws IOException Generic IO exception. - */ - public FeatureView getFeatureView(@NonNull String name, @NonNull Integer version) - throws FeatureStoreException, IOException { - return featureViewEngine.get(this, name, version); - } - - /** - * Get a feature view object from the selected feature store. - * - * 
-   * {@code
-   *        // get feature store handle
-   *        FeatureStore fs = HopsworksConnection.builder().build().getFeatureStore();
-   *        FeatureView fv = fs.getFeatureView("fv_name", 1);
-   * }
-   *
- * - * @param name Name of the feature view. - * @return FeatureView The feature view metadata object. - * @throws FeatureStoreException If unable to retrieve FeatureView from the feature store. - * @throws IOException Generic IO exception. - */ - public FeatureView getFeatureView(String name) throws FeatureStoreException, IOException { - LOGGER.info("VersionWarning: No version provided for getting feature view `" + name + "`, defaulting to `" - + DEFAULT_VERSION + "`."); - return getFeatureView(name, DEFAULT_VERSION); - } - - @Override - public Object getOrCreateFeatureView(String name, Query query, Integer version) - throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getOrCreateFeatureView(String name, Query query, Integer version, String description, - List labels) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getExternalFeatureGroup(@NonNull String name, @NonNull Integer version) - throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getExternalFeatureGroup(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public StorageConnector getStorageConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getHopsFsConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getExternalFeatureGroups(@NonNull String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object sql(String name) { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getJdbcConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getS3Connector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getRedshiftConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getSnowflakeConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getAdlsConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for beam"); - } - - @Override - public Object getKafkaConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getBigqueryConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getOnlineStorageConnector() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getGcsConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public TrainingDatasetBase getTrainingDataset(@NonNull String name, @NonNull Integer version) - throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public TrainingDatasetBase getTrainingDataset(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getTrainingDatasets(@NonNull String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } -} diff --git a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/FeatureView.java b/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/FeatureView.java deleted file mode 100644 index 48c54f127..000000000 --- a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/FeatureView.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.beam; - -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.FeatureViewBase; -import com.logicalclocks.hsfs.beam.constructor.Query; -import org.apache.beam.sdk.values.PCollection; - -import java.io.IOException; -import java.text.ParseException; -import java.util.Map; - -public class FeatureView extends FeatureViewBase> { - @Override - public void addTag(String name, Object value) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Map getTags() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getTag(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void deleteTag(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void addTrainingDatasetTag(Integer version, String name, Object value) - throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Map getTrainingDatasetTags(Integer version) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getTrainingDatasetTag(Integer version, String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void deleteTrainingDatasetTag(Integer version, String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void delete() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void clean(FeatureStore featureStore, String featureViewName, Integer featureViewVersion) - throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public FeatureView update(FeatureView other) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public String getBatchQuery() throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public String getBatchQuery(String startTime, String endTime) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public PCollection getBatchData() throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public PCollection getBatchData(String startTime, String endTime) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public PCollection getBatchData(String startTime, String endTime, Map readOptions) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getTrainingData(Integer version, Map readOptions) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getTrainTestSplit(Integer version, Map readOptions) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object getTrainValidationTestSplit(Integer version, Map readOptions) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void purgeTrainingData(Integer version) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void purgeAllTrainingData() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void deleteTrainingDataset(Integer version) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void deleteAllTrainingDatasets() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } -} diff --git a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/HopsworksConnection.java b/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/HopsworksConnection.java deleted file mode 100644 index 8b19103f5..000000000 --- a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/HopsworksConnection.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.beam; - -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.HopsworksConnectionBase; -import com.logicalclocks.hsfs.SecretStore; -import com.logicalclocks.hsfs.metadata.Credentials; -import com.logicalclocks.hsfs.metadata.HopsworksClient; -import com.logicalclocks.hsfs.metadata.HopsworksHttpClient; - -import lombok.Builder; - -import software.amazon.awssdk.regions.Region; - -import java.io.IOException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; - -public class HopsworksConnection extends HopsworksConnectionBase { - - @Builder - public HopsworksConnection(String host, int port, String project, Region region, SecretStore secretStore, - boolean hostnameVerification, String trustStorePath, String certPath, String apiKeyFilePath, - String apiKeyValue) throws IOException, FeatureStoreException, CertificateException, KeyStoreException, - NoSuchAlgorithmException { - this.host = host; - this.port = port; - this.project = getProjectName(project); - this.region = region; - this.secretStore = secretStore; - this.hostnameVerification = hostnameVerification; - this.trustStorePath = trustStorePath; - this.certPath = certPath; - this.apiKeyFilePath = apiKeyFilePath; - this.apiKeyValue = apiKeyValue; - - HopsworksClient.setupHopsworksClient(host, port, region, secretStore, - hostnameVerification, trustStorePath, this.apiKeyFilePath, this.apiKeyValue); - this.projectObj = getProject(); - HopsworksClient.getInstance().setProject(this.projectObj); - Credentials credentials = HopsworksClient.getInstance().getCredentials(); - HopsworksHttpClient hopsworksHttpClient = HopsworksClient.getInstance().getHopsworksHttpClient(); - hopsworksHttpClient.setTrustStorePath(credentials.gettStore()); - hopsworksHttpClient.setKeyStorePath(credentials.getkStore()); - hopsworksHttpClient.setCertKey(credentials.getPassword()); - HopsworksClient.getInstance().setHopsworksHttpClient(hopsworksHttpClient); - } - - /** - * Retrieve the project feature store. - * - * @return FeatureStore object. - * @throws IOException Generic IO exception. - * @throws FeatureStoreException If client is not connected to Hopsworks - */ - public FeatureStore getFeatureStore() throws IOException, FeatureStoreException { - return getFeatureStore(rewriteFeatureStoreName(project)); - } - - /** - * Retrieve a feature store based on name. The feature store needs to be shared with - * the connection's project. The name is the project name of the feature store. - * - * @param name the name of the feature store to get the handle for - * @return FeatureStore object. - * @throws IOException Generic IO exception. - * @throws FeatureStoreException If client is not connected to Hopsworks - */ - public FeatureStore getFeatureStore(String name) throws IOException, FeatureStoreException { - return featureStoreApi.get(rewriteFeatureStoreName(name), FeatureStore.class); - } -} diff --git a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/StreamFeatureGroup.java b/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/StreamFeatureGroup.java deleted file mode 100644 index 312890dff..000000000 --- a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/StreamFeatureGroup.java +++ /dev/null @@ -1,387 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.beam; - -import com.logicalclocks.hsfs.Feature; -import com.logicalclocks.hsfs.FeatureGroupBase; -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.HudiOperationType; -import com.logicalclocks.hsfs.JobConfiguration; -import com.logicalclocks.hsfs.StatisticsConfig; -import com.logicalclocks.hsfs.Storage; -import com.logicalclocks.hsfs.beam.engine.FeatureGroupEngine; -import com.logicalclocks.hsfs.beam.engine.BeamProducer; -import com.logicalclocks.hsfs.constructor.QueryBase; -import com.logicalclocks.hsfs.metadata.Statistics; -import lombok.Builder; -import lombok.NonNull; -import org.apache.beam.sdk.values.PCollection; - -import java.io.IOException; -import java.text.ParseException; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class StreamFeatureGroup extends FeatureGroupBase> { - - - protected FeatureGroupEngine featureGroupEngine = new FeatureGroupEngine(); - - @Builder - public StreamFeatureGroup(FeatureStore featureStore, @NonNull String name, Integer version, String description, - List primaryKeys, List partitionKeys, String hudiPrecombineKey, - boolean onlineEnabled, List features, - StatisticsConfig statisticsConfig, String onlineTopicName, String eventTime) { - this(); - this.featureStore = featureStore; - this.name = name; - this.version = version; - this.description = description; - this.primaryKeys = primaryKeys != null - ? primaryKeys.stream().map(String::toLowerCase).collect(Collectors.toList()) : null; - this.partitionKeys = partitionKeys != null - ? partitionKeys.stream().map(String::toLowerCase).collect(Collectors.toList()) : null; - this.hudiPrecombineKey = hudiPrecombineKey != null ? hudiPrecombineKey.toLowerCase() : null; - this.onlineEnabled = onlineEnabled; - this.features = features; - this.statisticsConfig = statisticsConfig != null ? statisticsConfig : new StatisticsConfig(); - this.onlineTopicName = onlineTopicName; - this.eventTime = eventTime; - } - - public StreamFeatureGroup() { - this.type = "streamFeatureGroupDTO"; - } - - // used for updates - public StreamFeatureGroup(Integer id, String description, List features) { - this(); - this.id = id; - this.description = description; - this.features = features; - } - - public StreamFeatureGroup(FeatureStore featureStore, int id) { - this(); - this.featureStore = featureStore; - this.id = id; - } - - @Override - public PCollection read() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public PCollection read(boolean online) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public PCollection read(Map readOptions) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public PCollection read(boolean online, Map readOptions) - throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public PCollection read(String wallclockTime) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public PCollection read(String wallclockTime, Map readOptions) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public QueryBase asOf(String wallclockTime) throws FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public QueryBase asOf(String wallclockTime, String excludeUntil) throws FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void show(int numRows) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void show(int numRows, boolean online) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void insert(PCollection featureData) throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void insert(PCollection featureData, Map writeOptions) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void insert(PCollection featureData, Storage storage) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void insert(PCollection featureData, boolean overwrite) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void insert(PCollection featureData, Storage storage, boolean overwrite) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void insert(PCollection featureData, boolean online, Map writeOptions) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void insert(PCollection featureData, HudiOperationType hudiOperationType) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void insert(PCollection featureData, Storage storage, boolean online, - HudiOperationType hudiOperationType, Map writeOptions) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void insert(PCollection featureData, JobConfiguration jobConfiguration) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void insert(PCollection featureData, boolean online, Map writeOptions, - JobConfiguration jobConfiguration) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void commitDeleteRecord(PCollection featureData) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void commitDeleteRecord(PCollection featureData, Map writeOptions) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Map> commitDetails() throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Map> commitDetails(Integer integer) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Map> commitDetails(String limit) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Map> commitDetails(String wallclockTime, Integer limit) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public QueryBase selectFeatures(List features) { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public QueryBase select(List features) { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public QueryBase selectAll() { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public QueryBase selectExceptFeatures(List features) { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public QueryBase selectExcept(List features) { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - /** - * Ingest a feature data to the online feature store using Beam Pipeline object. Currently, - * only org.apache.beam.sdk.values.Row types as feature data type are supported. - * - * 
-   * {@code
-   *        // get feature store handle
-   *        FeatureStore fs = HopsworksConnection.builder().build().getFeatureStore();
-   *
-   *        // get feature group handle
-   *        StreamFeatureGroup fg = fs.getStreamFeatureGroup("taxi_ride", 1);
-   *
-   *        // create Beam pipeline
-   *        Pipeline pipeline = Pipeline.create();
-   *        pipeline
-   *         .apply("read stream from the source", PubsubIO.readStrings().fromTopic(options.getInputTopic()))
-   *         .apply("Parse JSON to Beam Rows", JsonToRow.withSchema(schema))
-   *         .apply("insert streaming feature data", fg.insertStream());
-   * }
-   *
- * - * @return BeamProducer object, that can be wrapped inside Beam Pipeline `apply` method. - */ - public BeamProducer insertStream() throws Exception { - return featureGroupEngine.insertStream(this, null); - } - - public BeamProducer insertStream(Map writeOptions) throws Exception { - return featureGroupEngine.insertStream(this, writeOptions); - } - - @Override - public Object insertStream(PCollection featureData) throws Exception { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object insertStream(PCollection featureData, String queryName) throws Exception { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object insertStream(PCollection featureData, Map writeOptions) throws Exception { - return null; - } - - @Override - public Object insertStream(PCollection featureData, String queryName, Map writeOptions) - throws Exception { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object insertStream(PCollection featureData, String queryName, String outputMode) throws Exception { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object insertStream(PCollection featureData, String queryName, String outputMode, - String checkpointLocation) throws Exception { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object insertStream(PCollection featureData,String queryName, String outputMode, - boolean awaitTermination, Long timeout) throws Exception { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object insertStream(PCollection featureData, String queryName, String outputMode, - boolean awaitTermination, Long timeout, String checkpointLocation) throws Exception { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object insertStream(PCollection featureData, String queryName, String outputMode, - boolean awaitTermination, Long timeout, String checkpointLocation, Map writeOptions) - throws Exception { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object insertStream(PCollection featureData, String queryName, String outputMode, - boolean awaitTermination, String checkpointLocation) throws Exception { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object insertStream(PCollection featureData, String queryName, String outputMode, - boolean awaitTermination, Long timeout, String checkpointLocation, Map writeOptions, - JobConfiguration jobConfiguration) throws Exception { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void updateFeatures(List feature) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void updateFeatures(Feature feature) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void appendFeatures(List feature) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void appendFeatures(Feature feature) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Statistics computeStatistics() throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Statistics computeStatistics(String wallclockTime) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Statistics getStatistics() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } -} diff --git a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/constructor/Query.java b/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/constructor/Query.java deleted file mode 100644 index e3abe7df7..000000000 --- a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/constructor/Query.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.beam.constructor; - -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.Storage; -import com.logicalclocks.hsfs.beam.StreamFeatureGroup; -import com.logicalclocks.hsfs.constructor.QueryBase; - -import org.apache.beam.sdk.values.PCollection; - -import java.io.IOException; -import java.util.Map; - -public class Query extends QueryBase> { - @Override - public String sql() { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public String sql(Storage storage) { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object read() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object read(boolean online) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public Object read(boolean online, Map readOptions) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void show(int numRows) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public void show(boolean online, int numRows) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } -} diff --git a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/BeamEngine.java b/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/BeamEngine.java deleted file mode 100644 index 13ff573a1..000000000 --- a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/BeamEngine.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.beam.engine; - -import com.google.common.base.Strings; -import com.logicalclocks.hsfs.FeatureGroupBase; -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.StorageConnector; -import com.logicalclocks.hsfs.beam.StreamFeatureGroup; -import com.logicalclocks.hsfs.metadata.DatasetApi; -import com.logicalclocks.hsfs.engine.EngineBase; -import com.logicalclocks.hsfs.engine.FeatureGroupUtils; -import com.logicalclocks.hsfs.metadata.HopsworksInternalClient; -import org.apache.avro.Schema; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class BeamEngine extends EngineBase { - private static BeamEngine INSTANCE = null; - private FeatureGroupUtils featureGroupUtils = new FeatureGroupUtils(); - - public static synchronized BeamEngine getInstance() throws FeatureStoreException { - if (INSTANCE == null) { - INSTANCE = new BeamEngine(); - } - return INSTANCE; - } - - private BeamEngine() throws FeatureStoreException { - } - - public BeamProducer insertStream(StreamFeatureGroup streamFeatureGroup, Map writeOptions) - throws FeatureStoreException, IOException { - Map complexFeatureSchemas = new HashMap<>(); - for (String featureName: streamFeatureGroup.getComplexFeatures()) { - complexFeatureSchemas.put(featureName, - new Schema.Parser().parse(streamFeatureGroup.getFeatureAvroSchema(featureName))); - } - Schema deserializedEncodedSchema = new Schema.Parser().parse(streamFeatureGroup.getEncodedAvroSchema()); - - return new BeamProducer(streamFeatureGroup.getOnlineTopicName(), - getKafkaConfig(streamFeatureGroup, writeOptions), - streamFeatureGroup.getDeserializedAvroSchema(), deserializedEncodedSchema, complexFeatureSchemas, - streamFeatureGroup.getPrimaryKeys(), streamFeatureGroup); - } - - @Override - public String addFile(String filePath) throws IOException, FeatureStoreException { - if (Strings.isNullOrEmpty(filePath)) { - return filePath; - } - // this is used for unit testing - if (!filePath.startsWith("file://")) { - filePath = "hdfs://" + filePath; - } - String targetPath = System.getProperty("java.io.tmpdir") + filePath.substring(filePath.lastIndexOf("/")); - try (FileOutputStream outputStream = new FileOutputStream(targetPath)) { - outputStream.write(DatasetApi.readContent(filePath, featureGroupUtils.getDatasetType(filePath))); - } - return targetPath; - } - - @Override - public Map getKafkaConfig(FeatureGroupBase featureGroup, Map writeOptions) - throws FeatureStoreException, IOException { - boolean external = !(System.getProperties().containsKey(HopsworksInternalClient.REST_ENDPOINT_SYS) - || (writeOptions != null - && Boolean.parseBoolean(writeOptions.getOrDefault("internal_kafka", "false")))); - - StorageConnector.KafkaConnector storageConnector = - storageConnectorApi.getKafkaStorageConnector(featureGroup.getFeatureStore(), external); - storageConnector.setSslTruststoreLocation(addFile(storageConnector.getSslTruststoreLocation())); - storageConnector.setSslKeystoreLocation(addFile(storageConnector.getSslKeystoreLocation())); - - Map config = storageConnector.kafkaOptions(); - - if (writeOptions != null) { - config.putAll(writeOptions); - } - return config; - } -} diff --git a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/BeamKafkaProducer.java b/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/BeamKafkaProducer.java deleted file mode 100644 index c1bbd2748..000000000 --- a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/BeamKafkaProducer.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.beam.engine; - -import lombok.Setter; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.Future; - -public class BeamKafkaProducer extends KafkaProducer { - @Setter - private Map headerMap = new HashMap<>(); - - public BeamKafkaProducer(Map configs) { - super(configs); - } - - public Future send(ProducerRecord record) { - addHeaders(record); - return super.send(record); - } - - public Future send(ProducerRecord record, Callback callback) { - addHeaders(record); - return super.send(record, callback); - } - - private void addHeaders(ProducerRecord record) { - for (Map.Entry entry: headerMap.entrySet()) { - record.headers().add(entry.getKey(), entry.getValue()); - } - } -} diff --git a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/BeamProducer.java b/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/BeamProducer.java deleted file mode 100644 index e2b13e074..000000000 --- a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/BeamProducer.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.beam.engine; - -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.beam.StreamFeatureGroup; -import lombok.NonNull; - -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.EncoderFactory; - -import org.apache.beam.sdk.extensions.avro.coders.AvroCoder; -import org.apache.beam.sdk.extensions.avro.schemas.utils.AvroUtils; -import org.apache.beam.sdk.io.kafka.KafkaIO; -import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PDone; -import org.apache.beam.sdk.values.Row; - -import org.apache.kafka.common.config.SslConfigs; -import org.apache.kafka.common.serialization.StringSerializer; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -public class BeamProducer extends PTransform<@NonNull PCollection, @NonNull PDone> { - private String topic; - private Map properties; - private transient Schema schema; - private transient Schema encodedSchema; - private Map deserializedComplexFeatureSchemas; - private List primaryKeys; - private final Map headerMap = new HashMap<>(); - - public BeamProducer(String topic, Map properties, Schema schema, Schema encodedSchema, - Map deserializedComplexFeatureSchemas, List primaryKeys, - StreamFeatureGroup streamFeatureGroup) throws FeatureStoreException, IOException { - this.schema = schema; - this.encodedSchema = encodedSchema; - this.topic = topic; - this.properties = properties; - this.deserializedComplexFeatureSchemas = deserializedComplexFeatureSchemas; - this.primaryKeys = primaryKeys; - - headerMap.put("projectId", - String.valueOf(streamFeatureGroup.getFeatureStore().getProjectId()).getBytes(StandardCharsets.UTF_8)); - headerMap.put("featureGroupId", String.valueOf(streamFeatureGroup.getId()).getBytes(StandardCharsets.UTF_8)); - headerMap.put("subjectId", - String.valueOf(streamFeatureGroup.getSubject().getId()).getBytes(StandardCharsets.UTF_8)); - } - - @Override - public PDone expand(PCollection input) { - - PCollection featureGroupAvroRecord = input - .apply("Convert to avro generic record", ParDo.of(new DoFn() { - @ProcessElement - public void processElement(ProcessContext c) { - GenericRecord genericRecord = AvroUtils.toGenericRecord(c.element(), schema); - c.output(genericRecord); - } - })).setCoder(AvroCoder.of(GenericRecord.class, schema)); - - if (!deserializedComplexFeatureSchemas.keySet().isEmpty()) { - featureGroupAvroRecord = featureGroupAvroRecord - .apply("Serialize complex features", ParDo.of(new DoFn() { - @ProcessElement - public void processElement(ProcessContext c) throws IOException { - GenericRecord encodedRecord = new GenericData.Record(encodedSchema); - for (Schema.Field field: c.element().getSchema().getFields()) { - if (deserializedComplexFeatureSchemas.containsKey(field.name())) { - GenericDatumWriter complexFeatureDatumWriter = - new GenericDatumWriter<>(deserializedComplexFeatureSchemas.get(field.name())); - ByteArrayOutputStream complexFeatureByteArrayOutputStream = new ByteArrayOutputStream(); - complexFeatureByteArrayOutputStream.reset(); - BinaryEncoder complexFeatureBinaryEncoder = - new EncoderFactory().binaryEncoder(complexFeatureByteArrayOutputStream, null); - complexFeatureDatumWriter.write(field.name(), complexFeatureBinaryEncoder); - complexFeatureBinaryEncoder.flush(); - encodedRecord.put(field.name(), ByteBuffer.wrap(complexFeatureByteArrayOutputStream.toByteArray())); - } - } - c.output(encodedRecord); - } - })); - } - - return featureGroupAvroRecord.apply("Convert To KV of primaryKey:GenericRecord", - ParDo.of(new DoFn>() { - @ProcessElement - public void processElement(ProcessContext c) { - List primaryKeyValues = new ArrayList<>(); - for (String primaryKey: primaryKeys) { - primaryKeyValues.add(c.element().get(primaryKey).toString()); - } - c.output(KV.of(String.join(";", primaryKeyValues), c.element())); - } - }) - ) - .apply("Sync to online feature group kafka topic", KafkaIO.write() - .withBootstrapServers(properties.get("bootstrap.servers").toString()) - .withTopic(topic) - //.withProducerConfigUpdates(properties) - .withKeySerializer(StringSerializer.class) - .withValueSerializer(GenericAvroSerializer.class) - .withInputTimestamp() - .withProducerFactoryFn(props -> { - // copy jks files from resources to dataflow workers - try { - Path keyStorePath = Paths.get(properties.get(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG)); - InputStream keyStoreStream = Objects.requireNonNull(BeamProducer.class.getClassLoader() - .getResourceAsStream(keyStorePath.getFileName().toString())); - if (!Files.exists(keyStorePath)) { - Files.copy(keyStoreStream, keyStorePath, StandardCopyOption.REPLACE_EXISTING); - } - Path trustStorePath = Paths.get(properties.get(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG)); - InputStream trustStoreStream = Objects.requireNonNull(BeamProducer.class.getClassLoader() - .getResourceAsStream(trustStorePath.getFileName().toString())); - if (!Files.exists(trustStorePath)) { - Files.copy(trustStoreStream, trustStorePath, StandardCopyOption.REPLACE_EXISTING); - } - } catch (IOException e) { - e.printStackTrace(); - } - props.putAll(properties); - BeamKafkaProducer producer = new BeamKafkaProducer(props); - producer.setHeaderMap(headerMap); - return producer; - }) - ); - } -} diff --git a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/FeatureGroupEngine.java b/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/FeatureGroupEngine.java deleted file mode 100644 index c56fb2673..000000000 --- a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/FeatureGroupEngine.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.beam.engine; - -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.beam.FeatureStore; -import com.logicalclocks.hsfs.beam.StreamFeatureGroup; -import com.logicalclocks.hsfs.engine.FeatureGroupEngineBase; -import lombok.SneakyThrows; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class FeatureGroupEngine extends FeatureGroupEngineBase { - - @SneakyThrows - public BeamProducer insertStream(StreamFeatureGroup streamFeatureGroup, Map writeOptions) { - return BeamEngine.getInstance().insertStream(streamFeatureGroup, writeOptions); - } - - public StreamFeatureGroup getStreamFeatureGroup(FeatureStore featureStore, String fgName, Integer fgVersion) - throws IOException, FeatureStoreException { - StreamFeatureGroup[] streamFeatureGroups = - featureGroupApi.getInternal(featureStore, fgName, fgVersion, StreamFeatureGroup[].class); - - // There can be only one single feature group with a specific name and version in a feature store - // There has to be one otherwise an exception would have been thrown. - StreamFeatureGroup resultFg = streamFeatureGroups[0]; - resultFg.setFeatureStore(featureStore); - return resultFg; - } - - public List getStreamFeatureGroups(FeatureStore featureStore, String fgName) - throws FeatureStoreException, IOException { - StreamFeatureGroup[] streamFeatureGroups = - featureGroupApi.getInternal(featureStore, fgName, null, StreamFeatureGroup[].class); - - return Arrays.asList(streamFeatureGroups); - } -} diff --git a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/FeatureViewEngine.java b/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/FeatureViewEngine.java deleted file mode 100644 index 0402d4c9b..000000000 --- a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/FeatureViewEngine.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.beam.engine; - -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.beam.FeatureStore; -import com.logicalclocks.hsfs.beam.FeatureView; -import com.logicalclocks.hsfs.beam.StreamFeatureGroup; -import com.logicalclocks.hsfs.beam.constructor.Query; -import com.logicalclocks.hsfs.engine.FeatureViewEngineBase; -import org.apache.beam.sdk.values.PCollection; - -import java.io.IOException; -import java.util.Date; -import java.util.List; -import java.util.Map; - -public class FeatureViewEngine extends FeatureViewEngineBase> { - @Override - public FeatureView update(FeatureView featureView) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public FeatureView get(FeatureStore featureStore, String name, Integer version) - throws FeatureStoreException, IOException { - FeatureView featureView = get(featureStore, name, version, FeatureView.class); - featureView.setFeatureStore(featureStore); - return featureView; - } - - @Override - public Query getBatchQuery(FeatureView featureView, Date startTime, Date endTime, Boolean withLabels, - Integer trainingDataVersion) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public String getBatchQueryString(FeatureView featureView, Date startTime, Date endTime, Integer trainingDataVersion) - throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public FeatureView getOrCreateFeatureView(FeatureStore featureStore, String name, Integer version, Query query, - String description, List labels) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } - - @Override - public PCollection getBatchData(FeatureView featureView, Date startTime, Date endTime, - Map readOptions, Integer trainingDataVersion) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Beam"); - } -} diff --git a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/GenericAvroSerializer.java b/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/GenericAvroSerializer.java deleted file mode 100644 index 556426a47..000000000 --- a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/GenericAvroSerializer.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.beam.engine; - -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.reflect.ReflectDatumWriter; -import org.apache.kafka.common.serialization.Serializer; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class GenericAvroSerializer implements Serializer { - - @Override - public byte[] serialize(String topic, GenericRecord genericRecord) { - DatumWriter datumWriter = new ReflectDatumWriter<>(genericRecord.getSchema()); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - byteArrayOutputStream.reset(); - - List records = new ArrayList<>(); - records.add(genericRecord); - - BinaryEncoder binaryEncoder = new EncoderFactory().binaryEncoder(byteArrayOutputStream, null); - for (GenericRecord segment: records) { - try { - datumWriter.write(segment, binaryEncoder); - } catch (IOException e) { - e.printStackTrace(); - } - } - try { - binaryEncoder.flush(); - } catch (IOException e) { - e.printStackTrace(); - } - return byteArrayOutputStream.toByteArray(); - } - -} diff --git a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/KeySerializer.java b/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/KeySerializer.java deleted file mode 100644 index d0af9a7eb..000000000 --- a/hsfs/java/beam/src/main/java/com/logicalclocks/hsfs/beam/engine/KeySerializer.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.logicalclocks.hsfs.beam.engine; - -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.common.serialization.Serializer; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; - -public class KeySerializer implements Serializer { - - List primaryKeys; - - public KeySerializer(List primaryKeys) { - this.primaryKeys = primaryKeys; - } - - @Override - public byte[] serialize(String topic, GenericRecord record) { - List primaryKeyValues = new ArrayList<>(); - for (String primaryKey: this.primaryKeys) { - primaryKeyValues.add(record.get(primaryKey).toString()); - } - return String.join(";", primaryKeyValues).getBytes(StandardCharsets.UTF_8); - } -} diff --git a/hsfs/java/flink/pom.xml b/hsfs/java/flink/pom.xml deleted file mode 100644 index d2d7b87e0..000000000 --- a/hsfs/java/flink/pom.xml +++ /dev/null @@ -1,92 +0,0 @@ - - - - hsfs-parent - com.logicalclocks - 4.0.0-SNAPSHOT - - 4.0.0 - - hsfs-flink - - - 1.17.1.0 - 2.13.4.2 - - - - - com.logicalclocks - hsfs - ${project.version} - compile - - - com.fasterxml.jackson.core - * - - - javax.xml.bind - jaxb-api - - - com.databricks - * - - - org.scala-lang - * - - - - - - org.apache.flink - flink-streaming-java - ${flink.version} - provided - - - org.apache.flink - flink-shaded-hadoop2 - - - - - - - org.apache.flink - flink-connector-kafka - ${flink.version} - provided - - - org.apache.flink - flink-shaded-hadoop2 - - - - - - - org.apache.flink - flink-avro - ${flink.version} - - - org.apache.flink - flink-shaded-hadoop2 - - - - - - com.fasterxml.jackson.core - jackson-databind - ${fasterxml.version} - - - - diff --git a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/FeatureStore.java b/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/FeatureStore.java deleted file mode 100644 index 3ab0dfe24..000000000 --- a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/FeatureStore.java +++ /dev/null @@ -1,329 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.flink; - -import com.logicalclocks.hsfs.FeatureStoreBase; -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.StatisticsConfig; -import com.logicalclocks.hsfs.StorageConnector; -import com.logicalclocks.hsfs.TimeTravelFormat; -import com.logicalclocks.hsfs.TrainingDatasetBase; -import com.logicalclocks.hsfs.flink.constructor.Query; -import com.logicalclocks.hsfs.flink.engine.FeatureViewEngine; -import com.logicalclocks.hsfs.metadata.StorageConnectorApi; - -import com.logicalclocks.hsfs.flink.engine.FeatureGroupEngine; - -import lombok.NonNull; - -import java.io.IOException; -import java.util.List; - -public class FeatureStore extends FeatureStoreBase { - - private FeatureGroupEngine featureGroupEngine; - private FeatureViewEngine featureViewEngine; - - public FeatureStore() { - storageConnectorApi = new StorageConnectorApi(); - featureViewEngine = new FeatureViewEngine(); - featureGroupEngine = new FeatureGroupEngine(); - } - - @Override - public Object createFeatureGroup() { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getFeatureGroups(@NonNull String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getOrCreateFeatureGroup(String name, Integer version) throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - - @Override - public Object getOrCreateFeatureGroup(String name, Integer integer, List primaryKeys, - boolean onlineEnabled, String eventTime) throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getOrCreateFeatureGroup(String name, Integer version, List primaryKeys, - List partitionKeys, boolean onlineEnabled, String eventTime) throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getOrCreateFeatureGroup(String name, Integer version, String description, List primaryKeys, - List partitionKeys, String hudiPrecombineKey, boolean onlineEnabled, TimeTravelFormat timeTravelFormat, - StatisticsConfig statisticsConfig, String topicName, String notificationTopicName, String eventTime) { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - /** - * Get a stream feature group object from the feature store. - * - *

Getting a stream feature group metadata handle enables to interact with the feature group, - * such as read the data or use the `Query`-API to perform joins between feature groups and create feature - * views. - * - * 

-   * {@code
-   *        // get feature store handle
-   *        FeatureStore fs = HopsworksConnection.builder().build().getFeatureStore();
-   *        FeatureGroup fg = fs.getStreamFeatureGroup("electricity_prices", 1);
-   * }
-   *
- * - * @param name the name of the feature group - * @return StreamFeatureGroup The stream feature group metadata object. - * @throws FeatureStoreException If unable to retrieve feature group from the feature store. - * @throws IOException Generic IO exception. - */ - @Override - public StreamFeatureGroup getStreamFeatureGroup(String name) throws FeatureStoreException, IOException { - LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `" - + DEFAULT_VERSION + "`."); - return getStreamFeatureGroup(name, DEFAULT_VERSION); - } - - /** - * Get a stream feature group object from the feature store. - * - *

Getting a stream feature group metadata handle enables to interact with the feature group, - * such as read the data or use the `Query`-API to perform joins between feature groups and create feature - * views. - * - * 

-   * {@code
-   *        // get feature store handle
-   *        FeatureStore fs = HopsworksConnection.builder().build().getFeatureStore();
-   *        FeatureGroup fg = fs.getStreamFeatureGroup("electricity_prices", 1);
-   * }
-   *
- * - * @param name the name of the feature group - * @param version the version of the feature group - * @return StreamFeatureGroup The stream feature group metadata object. - * @throws FeatureStoreException If unable to retrieve feature group from the feature store. - * @throws IOException Generic IO exception. - */ - @Override - public StreamFeatureGroup getStreamFeatureGroup(@NonNull String name, @NonNull Integer version) - throws FeatureStoreException, IOException { - return featureGroupEngine.getStreamFeatureGroup(this, name, version); - } - - @Override - public StreamFeatureGroup.StreamFeatureGroupBuilder createStreamFeatureGroup() { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public StreamFeatureGroup getOrCreateStreamFeatureGroup(String name, Integer version) - throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public StreamFeatureGroup getOrCreateStreamFeatureGroup(String name, Integer version, List primaryKeys, - boolean onlineEnabled, String eventTime) - throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public StreamFeatureGroup getOrCreateStreamFeatureGroup(String name, Integer version, List primaryKeys, - List partitionKeys, boolean onlineEnabled, - String eventTime) throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public StreamFeatureGroup getOrCreateStreamFeatureGroup(String name, Integer version, String description, - List primaryKeys, List partitionKeys, - String hudiPrecombineKey, boolean onlineEnabled, - StatisticsConfig statisticsConfig, - String eventTime) - throws IOException, FeatureStoreException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object createExternalFeatureGroup() { - return null; - } - - @Override - public Object createFeatureView() { - return null; - } - - @Override - public StorageConnector getStorageConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getHopsFsConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getJdbcConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getOnlineStorageConnector() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getGcsConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getS3Connector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getRedshiftConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getSnowflakeConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getAdlsConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getKafkaConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getBigqueryConnector(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getExternalFeatureGroups(@NonNull String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object sql(String query) { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public TrainingDatasetBase getTrainingDataset(@NonNull String name, @NonNull Integer version) - throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public TrainingDatasetBase getTrainingDataset(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getTrainingDatasets(@NonNull String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public FeatureView getOrCreateFeatureView(String name, Query query, Integer version) - throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public FeatureView getOrCreateFeatureView(String name, Query query, Integer version, String description, - List labels) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - /** - * Get a feature view object from the selected feature store. - * - * 
-   * {@code
-   *        // get feature store handle
-   *        FeatureStore fs = HopsworksConnection.builder().build().getFeatureStore();
-   *        FeatureView fv = fs.getFeatureView("fv_name", 1);
-   * }
-   *
- * - * @param name Name of the feature view. - * @param version Version to get. - * @return FeatureView The feature view metadata object. - * @throws FeatureStoreException If unable to retrieve FeatureView from the feature store. - * @throws IOException Generic IO exception. - */ - public FeatureView getFeatureView(@NonNull String name, @NonNull Integer version) - throws FeatureStoreException, IOException { - return featureViewEngine.get(this, name, version); - } - - /** - * Get a feature view object from the selected feature store. - * - * 
-   * {@code
-   *        // get feature store handle
-   *        FeatureStore fs = HopsworksConnection.builder().build().getFeatureStore();
-   *        FeatureView fv = fs.getFeatureView("fv_name", 1);
-   * }
-   *
- * - * @param name Name of the feature view. - * @return FeatureView The feature view metadata object. - * @throws FeatureStoreException If unable to retrieve FeatureView from the feature store. - * @throws IOException Generic IO exception. - */ - public FeatureView getFeatureView(String name) throws FeatureStoreException, IOException { - LOGGER.info("VersionWarning: No version provided for getting feature view `" + name + "`, defaulting to `" - + DEFAULT_VERSION + "`."); - return getFeatureView(name, DEFAULT_VERSION); - } - - @Override - public Object getExternalFeatureGroup(@NonNull String name, @NonNull Integer version) - throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getExternalFeatureGroup(String name) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } -} diff --git a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/FeatureView.java b/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/FeatureView.java deleted file mode 100644 index e95578acd..000000000 --- a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/FeatureView.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.flink; - -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.FeatureViewBase; -import com.logicalclocks.hsfs.flink.constructor.Query; - -import org.apache.flink.streaming.api.datastream.DataStream; - -import lombok.NoArgsConstructor; - -import java.io.IOException; -import java.text.ParseException; - -import java.util.Map; - -@NoArgsConstructor -public class FeatureView extends FeatureViewBase> { - - @Override - public void addTag(String s, Object o) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Map getTags() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getTag(String s) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void deleteTag(String s) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void addTrainingDatasetTag(Integer integer, String s, Object o) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Map getTrainingDatasetTags(Integer integer) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getTrainingDatasetTag(Integer integer, String s) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void deleteTrainingDatasetTag(Integer integer, String s) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void delete() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void clean(FeatureStore featureStore, String s, Integer integer) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public FeatureView update(FeatureView featureView) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public String getBatchQuery() throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public String getBatchQuery(String s, String s1) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public DataStream getBatchData() throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public DataStream getBatchData(String s, String s1) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public DataStream getBatchData(String s, String s1, Map map) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getTrainingData(Integer integer, Map map) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getTrainTestSplit(Integer integer, Map map) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object getTrainValidationTestSplit(Integer integer, Map map) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void purgeTrainingData(Integer integer) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void purgeAllTrainingData() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void deleteTrainingDataset(Integer integer) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void deleteAllTrainingDatasets() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } -} diff --git a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/HopsworksConnection.java b/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/HopsworksConnection.java deleted file mode 100644 index 3d8d71d0f..000000000 --- a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/HopsworksConnection.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.flink; - -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.HopsworksConnectionBase; -import com.logicalclocks.hsfs.SecretStore; -import com.logicalclocks.hsfs.flink.engine.FlinkEngine; -import com.logicalclocks.hsfs.metadata.HopsworksClient; - -import com.logicalclocks.hsfs.metadata.HopsworksHttpClient; -import com.logicalclocks.hsfs.metadata.HopsworksInternalClient; -import lombok.Builder; - -import software.amazon.awssdk.regions.Region; - -import java.io.IOException; - -public class HopsworksConnection extends HopsworksConnectionBase { - - @Builder - public HopsworksConnection(String host, int port, String project, Region region, SecretStore secretStore, - boolean hostnameVerification, String trustStorePath, - String certPath, String apiKeyFilePath, String apiKeyValue) - throws IOException, FeatureStoreException { - this.host = host; - this.port = port; - this.project = getProjectName(project); - this.region = region; - this.secretStore = secretStore; - this.hostnameVerification = hostnameVerification; - this.trustStorePath = trustStorePath; - this.certPath = certPath; - this.apiKeyFilePath = apiKeyFilePath; - this.apiKeyValue = apiKeyValue; - - HopsworksClient.setupHopsworksClient(host, port, region, secretStore, - hostnameVerification, trustStorePath, this.apiKeyFilePath, this.apiKeyValue); - this.projectObj = getProject(); - HopsworksClient.getInstance().setProject(this.projectObj); - if (!System.getProperties().containsKey(HopsworksInternalClient.REST_ENDPOINT_SYS)) { - HopsworksHttpClient hopsworksHttpClient = HopsworksClient.getInstance().getHopsworksHttpClient(); - hopsworksHttpClient.setTrustStorePath(FlinkEngine.getInstance().getTrustStorePath()); - hopsworksHttpClient.setKeyStorePath(FlinkEngine.getInstance().getKeyStorePath()); - hopsworksHttpClient.setCertKey(HopsworksHttpClient.readCertKey(FlinkEngine.getInstance().getCertKey())); - HopsworksClient.getInstance().setHopsworksHttpClient(hopsworksHttpClient); - } - } - - /** - * Retrieve the project feature store. - * - * @return FeatureStore object. - * @throws IOException Generic IO exception. - * @throws FeatureStoreException If client is not connected to Hopsworks - */ - public FeatureStore getFeatureStore() throws IOException, FeatureStoreException { - return getFeatureStore(rewriteFeatureStoreName(project)); - } - - /** - * Retrieve a feature store based on name. The feature store needs to be shared with - * the connection's project. The name is the project name of the feature store. - * - * @param name the name of the feature store to get the handle for - * @return FeatureStore object. - * @throws IOException Generic IO exception. - * @throws FeatureStoreException If client is not connected to Hopsworks - */ - public FeatureStore getFeatureStore(String name) throws IOException, FeatureStoreException { - return featureStoreApi.get(rewriteFeatureStoreName(name), FeatureStore.class); - } -} diff --git a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/StreamFeatureGroup.java b/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/StreamFeatureGroup.java deleted file mode 100644 index 41b121d8b..000000000 --- a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/StreamFeatureGroup.java +++ /dev/null @@ -1,389 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.flink; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.logicalclocks.hsfs.Feature; -import com.logicalclocks.hsfs.FeatureGroupBase; -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.HudiOperationType; -import com.logicalclocks.hsfs.JobConfiguration; -import com.logicalclocks.hsfs.StatisticsConfig; -import com.logicalclocks.hsfs.Storage; -import com.logicalclocks.hsfs.constructor.QueryBase; - -import com.logicalclocks.hsfs.metadata.Statistics; - -import com.logicalclocks.hsfs.flink.engine.FeatureGroupEngine; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.NonNull; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamSink; - -import java.io.IOException; -import java.text.ParseException; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -@AllArgsConstructor -@JsonIgnoreProperties(ignoreUnknown = true) -public class StreamFeatureGroup extends FeatureGroupBase> { - - protected FeatureGroupEngine featureGroupEngine = new FeatureGroupEngine(); - - @Builder - public StreamFeatureGroup(FeatureStore featureStore, @NonNull String name, Integer version, String description, - List primaryKeys, List partitionKeys, String hudiPrecombineKey, - boolean onlineEnabled, List features, StatisticsConfig statisticsConfig, - String onlineTopicName, String topicName, String notificationTopicName, String eventTime) { - this(); - this.featureStore = featureStore; - this.name = name; - this.version = version; - this.description = description; - this.primaryKeys = primaryKeys != null - ? primaryKeys.stream().map(String::toLowerCase).collect(Collectors.toList()) : null; - this.partitionKeys = partitionKeys != null - ? partitionKeys.stream().map(String::toLowerCase).collect(Collectors.toList()) : null; - this.hudiPrecombineKey = hudiPrecombineKey != null ? hudiPrecombineKey.toLowerCase() : null; - this.onlineEnabled = onlineEnabled; - this.features = features; - this.statisticsConfig = statisticsConfig != null ? statisticsConfig : new StatisticsConfig(); - this.onlineTopicName = onlineTopicName; - this.topicName = topicName; - this.notificationTopicName = notificationTopicName; - this.eventTime = eventTime; - } - - public StreamFeatureGroup() { - this.type = "streamFeatureGroupDTO"; - } - - // used for updates - public StreamFeatureGroup(Integer id, String description, List features) { - this(); - this.id = id; - this.description = description; - this.features = features; - } - - public StreamFeatureGroup(FeatureStore featureStore, int id) { - this(); - this.featureStore = featureStore; - this.id = id; - } - - @Override - public DataStream read() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public DataStream read(boolean online) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public DataStream read(Map readOptions) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public DataStream read(boolean online, Map readOptions) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public DataStream read(String wallclockTime) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public DataStream read(String wallclockTime, Map readOptions) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public QueryBase asOf(String wallclockTime) throws FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public QueryBase asOf(String wallclockTime, String excludeUntil) throws FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void show(int numRows) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void show(int numRows, boolean online) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void insert(DataStream featureData) throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void insert(DataStream featureData, Map writeOptions) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void insert(DataStream featureData, Storage storage) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void insert(DataStream featureData, boolean overwrite) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void insert(DataStream featureData, Storage storage, boolean overwrite) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void insert(DataStream featureData, boolean online, Map writeOptions) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void insert(DataStream featureData, HudiOperationType hudiOperationType) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void insert(DataStream featureData, Storage storage, boolean online, HudiOperationType hudiOperationType, - Map writeOptions) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void insert(DataStream featureData, JobConfiguration jobConfiguration) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void insert(DataStream featureData, boolean online, Map writeOptions, - JobConfiguration jobConfiguration) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void commitDeleteRecord(DataStream featureData) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void commitDeleteRecord(DataStream featureData, Map writeOptions) - throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Map> commitDetails() throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Map> commitDetails(Integer integer) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Map> commitDetails(String limit) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Map> commitDetails(String wallclockTime, Integer limit) - throws IOException, FeatureStoreException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public QueryBase selectFeatures(List features) { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public QueryBase select(List features) { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public QueryBase selectAll() { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public QueryBase selectExceptFeatures(List features) { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public QueryBase selectExcept(List features) { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - /** - * Ingest a feature data to the online feature store using Flink DataStream API. Currently, only POJO - * types as feature data type are supported. - * - * 
-   * {@code
-   *        // get feature store handle
-   *        FeatureStore fs = HopsworksConnection.builder().build().getFeatureStore();
-   *
-   *        // get feature group handle
-   *        StreamFeatureGroup fg = fs.getStreamFeatureGroup("card_transactions", 1);
-   *
-   *        // read stream from the source and aggregate stream
-   *        DataStream aggregationStream =
-   *          env.fromSource(transactionSource, customWatermark, "Transaction Kafka Source")
-   *          .keyBy(r -> r.getCcNum())
-   *          .window(SlidingEventTimeWindows.of(Time.minutes(windowLength), Time.minutes(1)))
-   *          .aggregate(new TransactionCountAggregate());
-   *
-   *        // insert streaming feature data
-   *        fg.insertStream(featureData);
-   * }
-   *
- * - * @param featureData Features in Streaming Dataframe to be saved. - * @return DataStreamSink object. - */ - @Override - public DataStreamSink insertStream(DataStream featureData) throws Exception { - return featureGroupEngine.insertStream(this, featureData, null); - } - - @Override - public DataStreamSink insertStream(DataStream featureData, Map writeOptions) throws Exception { - return featureGroupEngine.insertStream(this, featureData, writeOptions); - } - - @Override - public Object insertStream(DataStream featureData, String queryName) throws Exception { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object insertStream(DataStream featureData, String queryName, Map writeOptions) - throws Exception { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object insertStream(DataStream featureData, String queryName, String outputMode) throws Exception { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object insertStream(DataStream featureData, String queryName, String outputMode, - String checkpointLocation) throws Exception { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object insertStream(DataStream featureData,String queryName, String outputMode, - boolean awaitTermination, Long timeout) throws Exception { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object insertStream(DataStream featureData, String queryName, String outputMode, - boolean awaitTermination, Long timeout, String checkpointLocation) - throws Exception { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object insertStream(DataStream featureData, String queryName, String outputMode, - boolean awaitTermination, Long timeout, String checkpointLocation, - Map writeOptions) throws Exception { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object insertStream(DataStream featureData, String queryName, String outputMode, boolean awaitTermination, - String checkpointLocation) throws Exception { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object insertStream(DataStream featureData, String queryName, String outputMode, boolean awaitTermination, - Long timeout, String checkpointLocation, Map writeOptions, - JobConfiguration jobConfiguration) throws Exception { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void updateFeatures(Feature feature) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void updateFeatures(List feature) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void appendFeatures(List feature) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void appendFeatures(Feature feature) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Statistics computeStatistics() throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Statistics computeStatistics(String wallclockTime) throws FeatureStoreException, IOException, ParseException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Statistics getStatistics() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } -} diff --git a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/constructor/FsQuery.java b/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/constructor/FsQuery.java deleted file mode 100644 index c9527119d..000000000 --- a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/constructor/FsQuery.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.flink.constructor; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.logicalclocks.hsfs.constructor.FsQueryBase; -import com.logicalclocks.hsfs.flink.StreamFeatureGroup; -import lombok.AllArgsConstructor; - -import java.util.Map; - -@JsonIgnoreProperties(ignoreUnknown = true) -@AllArgsConstructor -public class FsQuery extends FsQueryBase { - @Override - public void registerOnDemandFeatureGroups() { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void registerHudiFeatureGroups(Map readOptions) { - throw new UnsupportedOperationException("Not supported for Flink"); - } -} diff --git a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/constructor/Query.java b/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/constructor/Query.java deleted file mode 100644 index 4d1c85359..000000000 --- a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/constructor/Query.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.flink.constructor; - -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.Storage; -import com.logicalclocks.hsfs.constructor.QueryBase; - -import com.logicalclocks.hsfs.flink.StreamFeatureGroup; -import lombok.NoArgsConstructor; -import org.apache.flink.streaming.api.datastream.DataStream; - -import java.io.IOException; -import java.util.Map; - -@NoArgsConstructor -public class Query extends QueryBase> { - - @Override - public String sql() { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public String sql(Storage storage) { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object read() throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object read(boolean online) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public Object read(boolean online, Map readOptions) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void show(int i) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public void show(boolean online, int numRows) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } -} diff --git a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/FeatureGroupEngine.java b/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/FeatureGroupEngine.java deleted file mode 100644 index 7fe3d0b82..000000000 --- a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/FeatureGroupEngine.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.flink.engine; - -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.engine.FeatureGroupEngineBase; - -import com.logicalclocks.hsfs.flink.FeatureStore; -import com.logicalclocks.hsfs.flink.StreamFeatureGroup; -import lombok.SneakyThrows; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamSink; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -public class FeatureGroupEngine extends FeatureGroupEngineBase { - - @SneakyThrows - public DataStreamSink insertStream(StreamFeatureGroup streamFeatureGroup, DataStream featureData, - Map writeOptions) { - return FlinkEngine.getInstance().writeDataStream(streamFeatureGroup, featureData, writeOptions); - } - - public StreamFeatureGroup getStreamFeatureGroup(FeatureStore featureStore, String fgName, Integer fgVersion) - throws IOException, FeatureStoreException { - StreamFeatureGroup[] streamFeatureGroups = - featureGroupApi.getInternal(featureStore, fgName, fgVersion, StreamFeatureGroup[].class); - - // There can be only one single feature group with a specific name and version in a feature store - // There has to be one otherwise an exception would have been thrown. - StreamFeatureGroup resultFg = streamFeatureGroups[0]; - resultFg.setFeatureStore(featureStore); - return resultFg; - } - - public List getStreamFeatureGroups(FeatureStore featureStore, String fgName) - throws FeatureStoreException, IOException { - StreamFeatureGroup[] streamFeatureGroups = - featureGroupApi.getInternal(featureStore, fgName, null, StreamFeatureGroup[].class); - - return Arrays.asList(streamFeatureGroups); - } -} diff --git a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/FeatureViewEngine.java b/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/FeatureViewEngine.java deleted file mode 100644 index 6562b4ae4..000000000 --- a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/FeatureViewEngine.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.flink.engine; - -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.engine.FeatureViewEngineBase; - -import com.logicalclocks.hsfs.flink.StreamFeatureGroup; -import com.logicalclocks.hsfs.flink.constructor.Query; -import com.logicalclocks.hsfs.flink.FeatureView; -import com.logicalclocks.hsfs.flink.FeatureStore; - -import org.apache.flink.streaming.api.datastream.DataStream; - -import java.io.IOException; -import java.util.Date; -import java.util.List; -import java.util.Map; - -public class FeatureViewEngine extends FeatureViewEngineBase> { - - @Override - public FeatureView update(FeatureView featureView) throws FeatureStoreException, IOException { - featureViewApi.update(featureView, FeatureView.class); - return featureView; - } - - @Override - public FeatureView get(FeatureStore featureStore, String name, Integer version) - throws FeatureStoreException, IOException { - FeatureView featureView = get(featureStore, name, version, FeatureView.class); - featureView.setFeatureStore(featureStore); - return featureView; - } - - @Override - public Query getBatchQuery(FeatureView featureView, Date date, Date date1, Boolean withLabels, Integer integer) - throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public String getBatchQueryString(FeatureView featureView, Date startTime, Date endTime, Integer trainingDataVersion) - throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public FeatureView getOrCreateFeatureView(FeatureStore featureStore, String name, Integer version, Query query, - String description, List labels) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } - - @Override - public DataStream getBatchData(FeatureView featureView, Date startTime, Date endTime, Map readOptions, - Integer trainingDataVersion) throws FeatureStoreException, IOException { - throw new UnsupportedOperationException("Not supported for Flink"); - } -} diff --git a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/FlinkEngine.java b/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/FlinkEngine.java deleted file mode 100644 index 9e0645e96..000000000 --- a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/FlinkEngine.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.flink.engine; - -import com.google.common.base.Strings; -import com.logicalclocks.hsfs.FeatureGroupBase; -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.StorageConnector; -import com.logicalclocks.hsfs.engine.EngineBase; -import com.logicalclocks.hsfs.flink.StreamFeatureGroup; - -import com.logicalclocks.hsfs.metadata.HopsworksInternalClient; -import lombok.Getter; - -import org.apache.avro.generic.GenericRecord; -import org.apache.flink.configuration.ConfigOption; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.GlobalConfiguration; -import org.apache.flink.connector.base.DeliveryGuarantee; -import org.apache.flink.connector.kafka.sink.KafkaSink; -import org.apache.flink.core.fs.Path; -import org.apache.flink.formats.avro.typeutils.GenericRecordAvroTypeInfo; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.DataStreamSink; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.util.FileUtils; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import static org.apache.flink.configuration.ConfigOptions.key; - -public class FlinkEngine extends EngineBase { - private static FlinkEngine INSTANCE = null; - - public static synchronized FlinkEngine getInstance() throws FeatureStoreException { - if (INSTANCE == null) { - INSTANCE = new FlinkEngine(); - } - return INSTANCE; - } - - @Getter - private StreamExecutionEnvironment streamExecutionEnvironment; - - private final Configuration flinkConfig = GlobalConfiguration.loadConfiguration(); - private final ConfigOption keyStorePath = - key("flink.hadoop.hops.ssl.keystore.name") - .stringType() - .defaultValue("trustStore.jks") - .withDescription("path to keyStore.jks"); - private final ConfigOption trustStorePath = - key("flink.hadoop.hops.ssl.truststore.name") - .stringType() - .defaultValue("trustStore.jks") - .withDescription("path to trustStore.jks"); - private final ConfigOption materialPasswdPath = - key("flink.hadoop.hops.ssl.keystores.passwd.name") - .stringType() - .defaultValue("material_passwd") - .withDescription("path to material_passwd"); - - private FlinkEngine() throws FeatureStoreException { - streamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment(); - // Configure the streamExecutionEnvironment - streamExecutionEnvironment.getConfig().enableObjectReuse(); - } - - public DataStreamSink writeDataStream(StreamFeatureGroup streamFeatureGroup, DataStream dataStream, - Map writeOptions) throws FeatureStoreException, IOException { - - DataStream genericDataStream = (DataStream) dataStream; - Properties properties = new Properties(); - properties.putAll(getKafkaConfig(streamFeatureGroup, writeOptions)); - - KafkaSink sink = KafkaSink.builder() - .setBootstrapServers(properties.getProperty("bootstrap.servers")) - .setKafkaProducerConfig(properties) - .setRecordSerializer(new KafkaRecordSerializer(streamFeatureGroup)) - .setDeliverGuarantee(DeliveryGuarantee.AT_LEAST_ONCE) - .build(); - Map complexFeatureSchemas = new HashMap<>(); - for (String featureName: streamFeatureGroup.getComplexFeatures()) { - complexFeatureSchemas.put(featureName, streamFeatureGroup.getFeatureAvroSchema(featureName)); - } - - DataStream avroRecordDataStream = - genericDataStream.map(new PojoToAvroRecord( - streamFeatureGroup.getDeserializedAvroSchema(), - streamFeatureGroup.getDeserializedEncodedAvroSchema(), - complexFeatureSchemas)) - .returns( - new GenericRecordAvroTypeInfo(streamFeatureGroup.getDeserializedEncodedAvroSchema()) - ); - - return avroRecordDataStream.sinkTo(sink); - } - - @Override - public String addFile(String filePath) throws IOException { - if (Strings.isNullOrEmpty(filePath)) { - return filePath; - } - // this is used for unit testing - if (!filePath.startsWith("file://")) { - filePath = "hdfs://" + filePath; - } - String targetPath = FileUtils.getCurrentWorkingDirectory().toString() - + filePath.substring(filePath.lastIndexOf("/")); - FileUtils.copy(new Path(filePath), new Path(targetPath), false); - return targetPath; - } - - @Override - public Map getKafkaConfig(FeatureGroupBase featureGroup, Map writeOptions) - throws FeatureStoreException, IOException { - boolean external = !(System.getProperties().containsKey(HopsworksInternalClient.REST_ENDPOINT_SYS) - || (writeOptions != null - && Boolean.parseBoolean(writeOptions.getOrDefault("internal_kafka", "false")))); - - StorageConnector.KafkaConnector storageConnector = - storageConnectorApi.getKafkaStorageConnector(featureGroup.getFeatureStore(), external); - storageConnector.setSslTruststoreLocation(addFile(storageConnector.getSslTruststoreLocation())); - storageConnector.setSslKeystoreLocation(addFile(storageConnector.getSslKeystoreLocation())); - - Map config = storageConnector.kafkaOptions(); - - if (writeOptions != null) { - config.putAll(writeOptions); - } - config.put("enable.idempotence", "false"); - return config; - } - - public String getTrustStorePath() { - return flinkConfig.getString(trustStorePath); - } - - public String getKeyStorePath() { - return flinkConfig.getString(keyStorePath); - } - - public String getCertKey() { - return flinkConfig.getString(materialPasswdPath); - } -} diff --git a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/KafkaRecordSerializer.java b/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/KafkaRecordSerializer.java deleted file mode 100644 index b1729f75d..000000000 --- a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/KafkaRecordSerializer.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.flink.engine; - -import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.flink.StreamFeatureGroup; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.reflect.ReflectDatumWriter; -import org.apache.flink.api.common.serialization.SerializationSchema; -import org.apache.flink.connector.kafka.sink.KafkaRecordSerializationSchema; -import org.apache.kafka.clients.producer.ProducerRecord; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class KafkaRecordSerializer implements KafkaRecordSerializationSchema { - - private final String topic; - private final List primaryKeys; - private final Map headerMap = new HashMap<>(); - - KafkaRecordSerializer(StreamFeatureGroup streamFeatureGroup) throws FeatureStoreException, IOException { - this.topic = streamFeatureGroup.getOnlineTopicName(); - this.primaryKeys = streamFeatureGroup.getPrimaryKeys(); - - headerMap.put("projectId", - String.valueOf(streamFeatureGroup.getFeatureStore().getProjectId()).getBytes(StandardCharsets.UTF_8)); - headerMap.put("featureGroupId", String.valueOf(streamFeatureGroup.getId()).getBytes(StandardCharsets.UTF_8)); - headerMap.put("subjectId", - String.valueOf(streamFeatureGroup.getSubject().getId()).getBytes(StandardCharsets.UTF_8)); - } - - @Override - public void open(SerializationSchema.InitializationContext context, - KafkaRecordSerializationSchema.KafkaSinkContext sinkContext) { - // TODO not needed - } - - @Override - public ProducerRecord serialize(GenericRecord genericRecord, - KafkaRecordSerializationSchema.KafkaSinkContext context, - Long timestamp) { - byte[] key = this.serializeKey(genericRecord); - byte[] value = this.serializeValue(genericRecord); - ProducerRecord producerRecord = new ProducerRecord<>(topic, null, timestamp, key, value); - for (Map.Entry entry: headerMap.entrySet()) { - producerRecord.headers().add(entry.getKey(), entry.getValue()); - } - return producerRecord; - } - - public byte[] serializeKey(GenericRecord genericRecord) { - List primaryKeyValues = new ArrayList<>(); - for (String primaryKey: primaryKeys) { - primaryKeyValues.add(genericRecord.get(primaryKey).toString()); - } - return String.join(";", primaryKeyValues).getBytes(StandardCharsets.UTF_8); - } - - public byte[] serializeValue(GenericRecord genericRecord) { - DatumWriter datumWriter = new ReflectDatumWriter<>(genericRecord.getSchema()); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - byteArrayOutputStream.reset(); - - BinaryEncoder binaryEncoder = new EncoderFactory().binaryEncoder(byteArrayOutputStream, null); - try { - datumWriter.write(genericRecord, binaryEncoder); - binaryEncoder.flush(); - } catch (IOException e) { - e.printStackTrace(); - } - return byteArrayOutputStream.toByteArray(); - } -} \ No newline at end of file diff --git a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/PojoToAvroRecord.java b/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/PojoToAvroRecord.java deleted file mode 100644 index d2a37c26e..000000000 --- a/hsfs/java/flink/src/main/java/com/logicalclocks/hsfs/flink/engine/PojoToAvroRecord.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs.flink.engine; - -import org.apache.avro.Schema; -import org.apache.avro.SchemaValidationException; -import org.apache.avro.SchemaValidatorBuilder; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.EncoderFactory; -import org.apache.avro.reflect.ReflectData; -import org.apache.flink.api.common.functions.RichMapFunction; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.typeutils.ResultTypeQueryable; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.formats.avro.typeutils.GenericRecordAvroTypeInfo; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.lang.reflect.Field; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class PojoToAvroRecord extends RichMapFunction implements - ResultTypeQueryable { - - private final String schema; - private final String encodedSchema; - private final Map complexFeatureSchemas; - - // org.apache.avro.Schema$Field is not serializable. Create in open() and reused later on - private transient Schema deserializedSchema; - private transient Schema deserializedEncodedSchema; - private transient Map deserializedComplexFeatureSchemas; - private transient GenericRecordAvroTypeInfo producedType; - - public PojoToAvroRecord(Schema schema, Schema encodedSchema, Map complexFeatureSchemas) { - this.schema = schema.toString(); - this.encodedSchema = encodedSchema.toString(); - this.complexFeatureSchemas = complexFeatureSchemas; - } - - @Override - public GenericRecord map(T input) throws Exception { - - // validate - validatePojoAgainstSchema(input, this.deserializedSchema); - - // Create a new Avro record based on the given schema - GenericRecord record = new GenericData.Record(this.deserializedEncodedSchema); - // Get the fields of the POJO class populate fields of the Avro record - List fields = - Arrays.stream(input.getClass().getDeclaredFields()) - .filter(f -> f.getName().equals("SCHEMA$")) - .collect(Collectors.toList()); - if (!fields.isEmpty()) { - // it means POJO was generated from avro schema - Field schemaField = input.getClass().getDeclaredField("SCHEMA$"); - schemaField.setAccessible(true); - Schema fieldSchema = (Schema) schemaField.get(null); - for (Schema.Field field : fieldSchema.getFields()) { - String fieldName = field.name(); - Field pojoField = input.getClass().getDeclaredField(fieldName); - pojoField.setAccessible(true); - Object fieldValue = pojoField.get(input); - populateAvroRecord(record, fieldName, fieldValue); - } - } else { - for (Field field : fields) { - String fieldName = field.getName(); - Object fieldValue = field.get(input); - populateAvroRecord(record, fieldName, fieldValue); - } - } - return record; - } - - @Override - public void open(Configuration configuration) throws Exception { - super.open(configuration); - this.deserializedSchema = new Schema.Parser().parse(this.schema); - this.deserializedEncodedSchema = new Schema.Parser().parse(this.encodedSchema); - this.deserializedComplexFeatureSchemas = new HashMap<>(); - for (String featureName: this.complexFeatureSchemas.keySet()) { - deserializedComplexFeatureSchemas.put(featureName, - new Schema.Parser().parse(this.complexFeatureSchemas.get(featureName))); - } - this.producedType = new GenericRecordAvroTypeInfo(deserializedEncodedSchema); - } - - @Override - public TypeInformation getProducedType() { - return producedType; - } - - private void populateAvroRecord(GenericRecord record, String fieldName, Object fieldValue) throws IOException { - if (this.deserializedComplexFeatureSchemas.containsKey(fieldName)) { - GenericDatumWriter complexFeatureDatumWriter = - new GenericDatumWriter<>(this.deserializedComplexFeatureSchemas.get(fieldName)); - ByteArrayOutputStream complexFeatureByteArrayOutputStream = new ByteArrayOutputStream(); - complexFeatureByteArrayOutputStream.reset(); - BinaryEncoder complexFeatureBinaryEncoder = - new EncoderFactory().binaryEncoder(complexFeatureByteArrayOutputStream, null); - complexFeatureDatumWriter.write(fieldValue, complexFeatureBinaryEncoder); - complexFeatureBinaryEncoder.flush(); - record.put(fieldName, ByteBuffer.wrap(complexFeatureByteArrayOutputStream.toByteArray())); - complexFeatureByteArrayOutputStream.flush(); - complexFeatureByteArrayOutputStream.close(); - } else { - record.put(fieldName, fieldValue); - } - } - - private void validatePojoAgainstSchema(Object pojo, Schema avroSchema) throws SchemaValidationException { - Schema pojoSchema = ReflectData.get().getSchema(pojo.getClass()); - SchemaValidatorBuilder builder = new SchemaValidatorBuilder(); - builder.canReadStrategy().validateAll().validate(avroSchema, Collections.singletonList(pojoSchema)); - } -} diff --git a/hsfs/java/hsfs/pom.xml b/hsfs/java/hsfs/pom.xml deleted file mode 100644 index 56847be5d..000000000 --- a/hsfs/java/hsfs/pom.xml +++ /dev/null @@ -1,100 +0,0 @@ - - - - hsfs-parent - com.logicalclocks - 4.0.0-SNAPSHOT - - 4.0.0 - - hsfs - - - - 2.2.11 - - - - - - com.fasterxml.jackson.core - jackson-databind - ${fasterxml.jackson.databind.version} - provided - - - - - org.apache.avro - avro - ${avro.version} - - - - - javax.xml.bind - jaxb-api - ${javax.version} - provided - - - - software.amazon.awssdk - ssm - ${awssdk.version} - - - org.apache.httpcomponents - * - - - - - - software.amazon.awssdk - sts - ${awssdk.version} - - - org.apache.httpcomponents - * - - - - - - software.amazon.awssdk - secretsmanager - ${awssdk.version} - - - org.apache.httpcomponents - * - - - - - - com.databricks - dbutils-api_${scala-short.version} - ${dbutils.version} - provided - - - - - org.scala-lang - scala-library - ${scala.version} - - - - - commons-io - commons-io - 2.11.0 - - - diff --git a/hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/DataFormat.java b/hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/DataFormat.java deleted file mode 100644 index 89aab31c9..000000000 --- a/hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/DataFormat.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2020-2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public enum DataFormat { - @JsonProperty("csv") - CSV, - @JsonProperty("tsv") - TSV, - @JsonProperty("parquet") - PARQUET, - @JsonProperty("avro") - AVRO, - @JsonProperty("image") - IMAGE, - @JsonProperty("orc") - ORC, - @JsonProperty("tfrecords") - TFRECORDS, - @JsonProperty("tfrecord") - TFRECORD -} diff --git a/hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java b/hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java deleted file mode 100644 index 7990f2e42..000000000 --- a/hsfs/java/hsfs/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2022-2023. Hopsworks AB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * - * See the License for the specific language governing permissions and limitations under the License. - * - */ - -package com.logicalclocks.hsfs; - -import com.logicalclocks.hsfs.metadata.Option; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -import java.util.List; - -@NoArgsConstructor -@AllArgsConstructor -public class DeltaStreamerJobConf { - - @Getter - @Setter - private List