From ac33e7ffca6a342eb4bad7615a884306b8ad5dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Justen=20=28=40turicas=29?= Date: Thu, 22 Jun 2017 09:52:42 -0300 Subject: [PATCH] Change `rows {convert,query}` to be lazy --- rows/cli.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/rows/cli.py b/rows/cli.py index e72dfd96..be181e43 100755 --- a/rows/cli.py +++ b/rows/cli.py @@ -318,6 +318,7 @@ def print_(input_encoding, output_encoding, input_locale, output_locale, @click.option('--output-encoding') @click.option('--input-locale') @click.option('--output-locale') +@click.option('--samples', default=1000) @click.option('--verify-ssl', default=True, type=bool) @click.option('--output') @click.argument('query', required=True) @@ -341,26 +342,31 @@ def query(input_encoding, output_encoding, input_locale, output_locale, else: if input_locale is not None: with rows.locale_context(input_locale): - table = import_from_source(source, DEFAULT_INPUT_ENCODING) + table = import_from_source(source, DEFAULT_INPUT_ENCODING, + lazy=True, samples=samples) else: - table = import_from_source(source, DEFAULT_INPUT_ENCODING) + table = import_from_source(source, DEFAULT_INPUT_ENCODING, + lazy=True, samples=samples) sqlite_connection = sqlite3.Connection(':memory:') rows.export_to_sqlite(table, sqlite_connection, table_name='table1') - result = rows.import_from_sqlite(sqlite_connection, query=query) + result = rows.import_from_sqlite(sqlite_connection, query=query, + lazy=True, samples=samples) else: # TODO: if all sources are SQLite we can also optimize the import if input_locale is not None: with rows.locale_context(input_locale): tables = [_import_table(source, encoding=input_encoding, - verify_ssl=verify_ssl) + verify_ssl=verify_ssl, lazy=True, + samples=samples) for source in sources] else: tables = [_import_table(source, encoding=input_encoding, - verify_ssl=verify_ssl) + verify_ssl=verify_ssl, lazy=True, + samples=samples) for source in sources] sqlite_connection = sqlite3.Connection(':memory:') @@ -369,7 +375,8 @@ def query(input_encoding, output_encoding, input_locale, output_locale, sqlite_connection, table_name='table{}'.format(index)) - result = rows.import_from_sqlite(sqlite_connection, query=query) + result = rows.import_from_sqlite(sqlite_connection, query=query, + lazy=True, samples=samples) # TODO: may use sys.stdout.encoding if output_file = '-' output_encoding = output_encoding or sys.stdout.encoding or \