From 5713a47df0a801566c7c67af30b96488e761926f Mon Sep 17 00:00:00 2001 From: Emerson Rocha Date: Fri, 19 Feb 2021 05:16:45 -0300 Subject: [PATCH] hxl-on-databases (#10): quick test saving HXLated files on disk; not imported yet to databases; --- .gitignore | 8 +++++++- README.md | 5 +++++ tests/manual-tests.sh | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4eca3b7..6fc896d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,10 @@ hxlquickimport_samples/ *__pycache__* .Rhistory !tests/files/output/* -!tests/files/output/.gitkeep \ No newline at end of file +!tests/files/output/.gitkeep + +# hxlmeta souce storages +*.sqlite3 +hxlmeta-data/* +# ~/.config/hxl/hxlmeta-data.sqlite # SQLite database file +# ~/.config/hxl/hxlmeta-data/ # Directory containing HXLated CSV files \ No newline at end of file diff --git a/README.md b/README.md index f104bd4..dc9d45c 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,11 @@ tools. - https://github.com/ofajardo/pyreadr - Python for read SPSS, SAS and Stata - https://github.com/Roche/pyreadstat +- [meta issue] HXL and data directly from and to SQL databases #10 + - https://sqlite.org/inmemorydb.html + - https://github.com/wireservice/csvkit/blob/master/csvkit/utilities/csvsql.py + - https://stackoverflow.com/questions/32833145/advantages-of-an-in-memory-database-in-sqlite/32833770 + - Etc - Line break online https://www.joydeepdeb.com/tools/line-break.html diff --git a/tests/manual-tests.sh b/tests/manual-tests.sh index 5ee2a46..7eed8c5 100644 --- a/tests/manual-tests.sh +++ b/tests/manual-tests.sh @@ -94,6 +94,10 @@ hxlquickimporttab temp/titanic.tab | head # @see https://docs.sqlalchemy.org/en/13/dialects/ # @see https://github.com/wireservice/csvkit/blob/master/csvkit/utilities/csvsql.py +# TODO: Create an local PostgreSQL database +# @see https://csvkit.readthedocs.io/en/latest/scripts/csvsql.html +# createdb hxltest + # Generate create schema csvsql tests/files/iris_hxlated-csv.csv --dialect postgresql --skip-lines 1 csvsql tests/files/iris_hxlated-csv.csv --dialect mysql --skip-lines 1 @@ -132,6 +136,42 @@ csvsql tests/files/iris_hxlated-csv.csv --dialect mssql --skip-lines 1 # Insert data directly on the server csvsql tests/files/iris_hxlated-csv.csv --db 'postgresql://postgres:password@localhost/hxltest' --skip-lines 1 +#### Build hxlmeta local storage _______________________________________________ +# @see https://github.com/EticaAI/HXL-Data-Science-file-formats/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc +# Note: these hxlmeta examples may not actually be used by the HXLMeta Class +# but could at least be used to test importing/exporting from databases + +# @see https://github.com/harelba/q/blob/master/test/BENCHMARK.md + +# TODO hxl2example --remove-headers example.com/dataset hxlmeta-data/HXLMeta_DataType.hxl.csv + +### HXLMeta_DataType, HXLMeta_StorageType, HXLMeta_StatisticalType +hxl2example https://docs.google.com/spreadsheets/d/1hGUxMN2ywWNv8ONQ59Pp9Q4nG-eTRnAs0SyWunFZUDg/edit#gid=214068544 hxlmeta-data/HXLMeta_DataType.hxl.csv +hxl2example https://docs.google.com/spreadsheets/d/1vFkBSharAEg5g5K2u_iDLCBvpWWPqpzC1hcL6QpFNZY/edit#gid=211012023 hxlmeta-data/HXLMeta_StorageType.hxl.csv +hxl2example https://docs.google.com/spreadsheets/d/1vFkBSharAEg5g5K2u_iDLCBvpWWPqpzC1hcL6QpFNZY/edit#gid=1566300457 hxlmeta-data/HXLMeta_StatisticalType.hxl.csv +hxl2example https://docs.google.com/spreadsheets/d/1vFkBSharAEg5g5K2u_iDLCBvpWWPqpzC1hcL6QpFNZY/edit#gid=1053765950 hxlmeta-data/HXLMeta_LevelType.hxl.csv +hxl2example https://docs.google.com/spreadsheets/d/1vFkBSharAEg5g5K2u_iDLCBvpWWPqpzC1hcL6QpFNZY/edit#gid=617579056 hxlmeta-data/HXLMeta_UsageType.hxl.csv + +### HXL-CPLP-Vocab_Bool -------------------------------------------------------- +# @see https://github.com/HXL-CPLP/forum/issues/49 +hxl2example https://docs.google.com/spreadsheets/d/1hGUxMN2ywWNv8ONQ59Pp9Q4nG-eTRnAs0SyWunFZUDg/edit#gid=214068544 hxlmeta-data/HXL-CPLP-Vocab_Bool.hxl.csv + +### UNOCHA-countries-territories ----------------------------------------------- +# https://vocabulary.unocha.org/ +hxl2example https://docs.google.com/spreadsheets/d/1NjSI2LaS3SqbgYc0HdD8oIb7lofGtiHgoKKATCpwVdY/edit#gid=1088874596 hxlmeta-data/UNOCHA_countries-territories.hxl.csv + +### HXL-CPLP-FOD_languages ----------------------------------------------------- +hxl2example https://docs.google.com/spreadsheets/d/12k4BWqq5c3mV9ihQscPIwtuDa_QRB-iFohO7dXSSptI/edit#gid=0 hxlmeta-data/HXL-CPLP-FOD_languages.hxl.csv + +### HXLStandard_Data-types -------------------------------------------------- +hxl2example https://docs.google.com/spreadsheets/d/1En9FlmM8PrbTWgl3UHPF_MXnJ6ziVZFhBbojSJzBdLI/edit#gid=1881622062 hxlmeta-data/HXLStandard_Data-types.hxl.csv + +### HXLStandard_Core-hashtags -------------------------------------------------- +hxl2example https://docs.google.com/spreadsheets/d/1En9FlmM8PrbTWgl3UHPF_MXnJ6ziVZFhBbojSJzBdLI/edit#gid=319251406 hxlmeta-data/HXLStandard_Core-hashtags.hxl.csv + +### HXLStandard_Core-attributes ------------------------------------------------ +hxl2example https://docs.google.com/spreadsheets/d/1En9FlmM8PrbTWgl3UHPF_MXnJ6ziVZFhBbojSJzBdLI/edit#gid=1810309357 hxlmeta-data/HXLStandard_Core-attributes.hxl.csv + #### Ignore after this part ___________________________________________________ # fititnt@bravo:/workspace/data/brasil_inep_microdados-enem-2019/DADOS$ head -n 1000 MICRODADOS_ENEM_2019.csv > MICRODADOS_ENEM_2019_head-n-1000.csv hxlquickimport hxlquickimport_samples/MICRODADOS_ENEM_2019_head-n-1000.csv | hxl2tab