Introduce limit parameter in QES read method

[JanusAsmussen]

This PR introduces the ability to limit the results returned by the QES.

As part of this enhancement, the AstraClient has been refactored. The most notable change is the replacement of the concat method for MetaFrames with a call to itertools.chain.from_iterable

[JanusAsmussen]

create_package

[github-actions]

Created package with number v3.2.6a485.dev1

[github-actions]

Coverage Report

File	Stmts	Miss	Cover	Missing
adapta/connectors/service_bus
__init__.py	1	1	0%	18
_connector.py	17	17	0%	19–61
adapta/logs
_async_logger.py	81	3	96%	55, 78–79
_base.py	55	6	89%	35, 41, 44, 94–97
_internal_logger.py	109	6	94%	276–286
adapta/logs/handlers
datadog_api_handler.py	109	31	72%	89, 106–113, 124, 136–152, 161–195, 204, 210, 238
adapta/metrics/providers
datadog_provider.py	43	43	0%	19–147
adapta/ml
__init__.py	1	1	0%	19
_model.py	10	10	0%	17–42
adapta/ml/mlflow
__init__.py	2	2	0%	17–18
_client.py	42	42	0%	19–142
_functions.py	42	42	0%	17–111
adapta/process_communication
_models.py	33	6	82%	90–96
adapta/security/clients
__init__.py	26	12	54%	27–28, 34–35, 41–42, 48–49, 53–54, 58–59
_azure_client.py	73	50	32%	42, 55–65, 75–78, 81, 84–86, 95–153, 156, 159–197
adapta/security/clients/aws
_aws_client.py	38	17	55%	37–40, 47, 57, 63, 75, 100–108, 114–117
_aws_credentials.py	73	30	59%	60–79, 83, 87, 91, 95, 99, 108–112, 116, 120, 124, 128, 132
adapta/security/clients/hashicorp_vault
hashicorp_vault_client.py	31	4	87%	46, 87, 91, 95
kubernetes_client.py	21	5	76%	45–48, 67–68
oidc_client.py	45	20	56%	33–62, 80–83, 92
token_client.py	17	7	59%	42–45, 52–53, 56, 59
adapta/storage/blob
azure_storage_client.py	119	51	57%	71–78, 88, 95–96, 99–105, 127–128, 131–156, 159, 175–188, 196–200, 219, 240–242, 251, 267–271, 281–283, 286–305, 312
local_storage_client.py	54	7	87%	42, 67, 74, 87, 101, 104, 107
s3_storage_client.py	117	77	34%	57, 61, 68–80, 91–96, 117–125, 133–134, 148–165, 181–185, 203–212, 222–243, 254–275
adapta/storage/cache
redis_cache.py	37	37	0%	19–107
adapta/storage/database/v2
azure_sql.py	34	34	0%	21–140
odbc.py	73	73	0%	21–219
snowflake_sql.py	69	69	0%	6–228
trino_sql.py	39	39	0%	21–127
adapta/storage/database/v2/models
__init__.py	1	1	0%	19
_models.py	11	11	0%	20–54
adapta/storage/database/v3
azure_sql.py	32	20	38%	55, 70, 84–122, 132
odbc.py	72	11	85%	94–104, 114, 125–126, 140, 149–155, 182
snowflake_sql.py	77	23	70%	64–78, 93–95, 106–115, 142, 178, 193–196
trino_sql.py	38	38	0%	20–119
adapta/storage/delta_lake/v2
_functions.py	68	45	34%	73–106, 153–168, 210–294
adapta/storage/delta_lake/v3
_functions.py	68	13	81%	74, 88, 161, 166, 168, 170, 226, 236, 246–256, 262, 297
adapta/storage/distributed_object_store/v2/datastax_astra
_models.py	19	9	53%	54–59, 62, 65, 68
astra_client.py	196	140	29%	40–43, 113–137, 143–184, 190–191, 197–198, 201, 211, 223–224, 233, 273–361, 369, 393–520, 530, 541–556, 579–594, 616–632, 656–673
adapta/storage/distributed_object_store/v3/datastax_astra
_model_mappers.py	159	22	86%	64, 130, 136, 138, 142, 148, 152–153, 171–174, 205, 262–272, 357, 398–410, 442
_models.py	45	31	31%	63–69, 72, 75, 78–122, 125
astra_client.py	142	70	51%	36–39, 134–175, 181–182, 188–189, 192, 202, 214–215, 224, 270–342, 358, 370, 381–400, 421–437, 459–479, 504–516
adapta/storage/models
astra.py	35	10	71%	37, 40, 44, 59–61, 64, 70–73
aws.py	36	11	69%	36–39, 46–49, 57–59, 82, 91
azure.py	60	17	72%	32, 36–40, 47, 67, 70–71, 74–75, 89–93, 100, 113, 120, 123–124, 127
filter_expression.py	125	5	96%	55, 183–184, 238, 329
hive.py	57	25	56%	37, 41, 44, 92–99, 111, 114–115, 124–172, 175
local.py	21	4	81%	31, 35, 38, 50
adapta/storage/query_enabled_store
_models.py	65	13	80%	76, 138–142, 148–151, 157–158, 164–165, 171
_qes_astra.py	57	13	77%	65–66, 78, 83–97, 108–111
_qes_delta.py	39	4	90%	33, 60, 76, 85
adapta/storage/secrets
azure_secret_client.py	20	20	0%	19–66
adapta/utils
_common.py	95	15	84%	36–37, 70–83, 94, 122, 142, 162, 245
concurrent_task_runner.py	27	1	96%	109
adapta/utils/data_structures
_functions.py	34	1	97%	134
adapta/utils/decorators
_logging.py	41	1	98%	32
_rate_limit.py	25	1	96%	58
adapta/utils/python_typing
_functions.py	11	1	91%	24
tests
test_filtering_api.py	32	2	94%	197–198
test_utils.py	166	1	99%	371
test_vault_client.py	80	18	78%	33–35, 40–42, 47–51, 56–57, 62–66
TOTAL	4541	1339	71%

Tests	Skipped	Failures	Errors	Time
210	5 💤	0 ❌	0 🔥	1m 2s ⏱️

[github-actions]

Coverage Report

File	Stmts	Miss	Cover	Missing
adapta/connectors/service_bus
__init__.py	1	1	0%	18
_connector.py	17	17	0%	19–61
adapta/logs
_async_logger.py	81	3	96%	55, 78–79
_base.py	55	6	89%	35, 41, 44, 94–97
_internal_logger.py	109	6	94%	276–286
adapta/logs/handlers
datadog_api_handler.py	109	31	72%	89, 106–113, 124, 136–152, 161–195, 204, 210, 238
adapta/metrics/providers
datadog_provider.py	43	43	0%	19–147
adapta/ml
__init__.py	1	1	0%	19
_model.py	10	10	0%	17–42
adapta/ml/mlflow
__init__.py	2	2	0%	17–18
_client.py	42	42	0%	19–142
_functions.py	42	42	0%	17–111
adapta/process_communication
_models.py	33	6	82%	90–96
adapta/security/clients
__init__.py	26	12	54%	27–28, 34–35, 41–42, 48–49, 53–54, 58–59
_azure_client.py	73	50	32%	42, 55–65, 75–78, 81, 84–86, 95–153, 156, 159–197
adapta/security/clients/aws
_aws_client.py	38	17	55%	37–40, 47, 57, 63, 75, 100–108, 114–117
_aws_credentials.py	73	30	59%	60–79, 83, 87, 91, 95, 99, 108–112, 116, 120, 124, 128, 132
adapta/security/clients/hashicorp_vault
hashicorp_vault_client.py	31	4	87%	46, 87, 91, 95
kubernetes_client.py	21	5	76%	45–48, 67–68
oidc_client.py	45	20	56%	33–62, 80–83, 92
token_client.py	17	7	59%	42–45, 52–53, 56, 59
adapta/storage/blob
azure_storage_client.py	118	50	58%	71–78, 88, 95–96, 99–105, 127–128, 131–156, 159, 175–188, 196–200, 219, 240–242, 251, 267–271, 281–283, 286–305, 312
local_storage_client.py	54	7	87%	42, 67, 74, 87, 101, 104, 107
s3_storage_client.py	117	77	34%	57, 61, 68–80, 91–96, 117–125, 133–134, 148–165, 181–185, 203–212, 222–243, 254–275
adapta/storage/cache
redis_cache.py	37	37	0%	19–107
adapta/storage/database/v2
azure_sql.py	34	34	0%	21–140
odbc.py	73	73	0%	21–219
snowflake_sql.py	69	69	0%	6–228
trino_sql.py	39	39	0%	21–127
adapta/storage/database/v2/models
__init__.py	1	1	0%	19
_models.py	11	11	0%	20–54
adapta/storage/database/v3
azure_sql.py	32	20	38%	55, 70, 84–122, 132
odbc.py	72	11	85%	94–104, 114, 125–126, 140, 149–155, 182
snowflake_sql.py	77	23	70%	64–78, 93–95, 106–115, 142, 178, 193–196
trino_sql.py	38	38	0%	20–119
adapta/storage/delta_lake/v2
_functions.py	68	45	34%	73–106, 153–168, 210–294
adapta/storage/delta_lake/v3
_functions.py	68	13	81%	74, 88, 161, 166, 168, 170, 226, 236, 246–256, 262, 297
adapta/storage/distributed_object_store/v2/datastax_astra
_models.py	19	9	53%	54–59, 62, 65, 68
astra_client.py	196	140	29%	40–43, 113–137, 143–184, 190–191, 197–198, 201, 211, 223–224, 233, 273–361, 369, 393–520, 530, 541–556, 579–594, 616–632, 656–673
adapta/storage/distributed_object_store/v3/datastax_astra
_model_mappers.py	159	22	86%	64, 130, 136, 138, 142, 148, 152–153, 171–174, 205, 262–272, 357, 398–410, 442
_models.py	45	31	31%	63–69, 72, 75, 78–122, 125
astra_client.py	142	70	51%	36–39, 134–175, 181–182, 188–189, 192, 202, 214–215, 224, 270–342, 358, 370, 381–400, 421–437, 459–479, 504–516
adapta/storage/models
astra.py	35	10	71%	37, 40, 44, 59–61, 64, 70–73
aws.py	36	11	69%	36–39, 46–49, 57–59, 82, 91
azure.py	60	17	72%	32, 36–40, 47, 67, 70–71, 74–75, 89–93, 100, 113, 120, 123–124, 127
filter_expression.py	125	5	96%	55, 183–184, 238, 329
hive.py	57	25	56%	37, 41, 44, 92–99, 111, 114–115, 124–172, 175
local.py	21	4	81%	31, 35, 38, 50
adapta/storage/query_enabled_store
_models.py	65	13	80%	76, 138–142, 148–151, 157–158, 164–165, 171
_qes_astra.py	57	13	77%	65–66, 78, 83–97, 108–111
_qes_delta.py	39	4	90%	33, 60, 76, 85
adapta/storage/secrets
azure_secret_client.py	20	20	0%	19–66
adapta/utils
_common.py	95	15	84%	36–37, 70–83, 94, 122, 142, 162, 245
concurrent_task_runner.py	27	1	96%	109
adapta/utils/data_structures
_functions.py	34	1	97%	134
adapta/utils/decorators
_logging.py	41	1	98%	32
_rate_limit.py	25	1	96%	58
adapta/utils/python_typing
_functions.py	11	3	73%	7–9, 22
tests
test_filtering_api.py	32	2	94%	197–198
test_python_typing_functions.py	11	2	82%	38–39
test_utils.py	166	1	99%	371
test_vault_client.py	80	18	78%	33–35, 40–42, 47–51, 56–57, 62–66
TOTAL	4539	1342	70%

Tests	Skipped	Failures	Errors	Time
210	6 💤	0 ❌	0 🔥	1m 2s ⏱️

[JanusAsmussen]

create_package

[github-actions]

Created package with number v3.2.6a485.dev5

[george-zubrienko]

I'm ok with this, only thing I didn't get is in the comment. Should be good to go after that one is resolved

[george-zubrienko]

hmm but we did apply the limit in the query already? why do we apply it twice here?