-
Notifications
You must be signed in to change notification settings - Fork 2
/
notes.txt
331 lines (197 loc) · 6.92 KB
/
notes.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
################################################################################
## Notes
################################################################################
o. dat/ file generation
- Run generate_csv_files.R to make some files.
- Run alter_csv_files.sh to make some rows that have errors or other oddities to discover.
o. Create a sqlite database
- in alter_csv_files.sh
o. Things to do:
#### csvlook
# Looking at a file without these tools
cat dat/mtcars_001.csv
less dat/mtcars_001.csv
# A bit better
cat dat/mtcars_001.csv | column -t -s,
# ---
# look at a file with some formatting
csvlook dat/mtcars_001.csv
# Add line numbers
csvlook -l dat/mtcars_001.csv
# long file
csvlook dat/long_flights.csv
# Better if you could page through it
csvlook dat/long_flights.csv | less
# Look at a wide file ... gets ugly
csvlook dat/wide_mtcars.csv
# Can make look nicer with `less'
csvlook dat/wide_mtcars.csv | less -S
# Can issue `-S' if you forgot at the outset
# .. after starting, type `-S' and <enter> to not wrap lines
csvlook dat/wide_mtcars.csv | less
#### csvstats
# Check out what a file looks like:
csvstat dat/mtcars_001.csv
# Check out what a file looks like, but with a `less' pager
csvstat dat/mtcars_001.csv | less
# Check out what a file looks like, check for nulls
csvstat dat/mtcars_001.csv --nulls
# File that does have some nulls
csvstat dat/mtcars_blank-select_cells.csv --nulls
csvlook dat/mtcars_blank-select_cells.csv
csvstat dat/wide_mtcars.csv --nulls
csvlook dat/wide_mtcars.csv
csvstat dat/wide_flights_dest_month.csv --nulls
csvlook dat/wide_flights_dest_month.csv
# Check out what a file looks like, check for max
csvstat dat/mtcars_001.csv --max
# Check out what a file looks like, row count
csvstat dat/mtcars_001.csv --count
# Check out what a file looks like, row count
csvstat dat/mtcars_001.csv --freq
#### csvclean
# This has no errors
csvclean -n dat/mtcars_001.csv
# Has errors, see on stdout
csvclean -n dat/mtcars_too-many-fields.csv
# output errors to file for reading later on
csvclean dat/mtcars_too-many-fields.csv
#### in2csv
# Extract table data from excel
in2csv --sheet "mtcars-s1" dat/xlsx_mtcars.xlsx
in2csv --sheet "mtcars-s1" dat/xlsx_mtcars.xlsx | csvlook
# Extract table data from excel
# Be careful that the table starts in A1, or you have to do some pre-cleaning of the data
in2csv --sheet "mtcars-s2" dat/xlsx_mtcars.xlsx
in2csv --sheet "mtcars-s2" dat/xlsx_mtcars.xlsx | csvlook
# Convert from pipes to commas
less dat/mtcars_pipes.csv
in2csv dat/mtcars_pipes.csv
in2csv -d\| dat/mtcars_pipes.csv # Alternate explicit designation of pipe as the delimiter
# Convert from tabs to commas
less dat/mtcars_tabs.csv
in2csv dat/mtcars_tabs.csv
in2csv -t dat/mtcars_tabs.csv # -t specifies that the delimiter is a tab character
# (Advanced) json file
cat dat/issues.json | less
in2csv -f json -v dat/issues.json | csvlook | less
#### csvjson
# Convert
csvjson dat/mtcars_001.csv
#### csvstack
# Stack two files
csvstack dat/mtcars_001.csv dat/mtcars_002.csv
csvstack dat/mtcars_001.csv dat/mtcars_002.csv | csvlook
csvstack dat/mtcars_001.csv dat/mtcars_002.csv | csvlook | less
# Stack a set of files
csvstack dat/mtcars_00*.csv
csvstack dat/mtcars_00*.csv | csvlook | less
# Add the filenames to the csv output so you know where it came from
csvstack --filenames dat/mtcars_00*.csv | csvlook | less
# Doesn't read header names, so column names have to be in same order
csvstack dat/mtcars_001.csv dat/mtcars_col-order-shuffle.csv | csvlook | less
#### sql2csv
# Look at the database:
sqlite3 dat/mtcars.db
.tables
select * from mtcarstbl;
# extract from a database
sql2csv --db "sqlite:///dat/mtcars.db" --query "select * from mtcarstbl"
sql2csv --db "sqlite:///dat/mtcars.db" --query "select * from mtcarstbl" | csvlook
#### csvsql
# Insert into db
sqlite3 dat/mtcars.db
.tables
drop table test;
csvsql --db "sqlite:///dat/mtcars.db" --table "test" --insert dat/mtcars_001.csv
csvsql --db "sqlite:///dat/mtcars.db" --table "test" --insert dat/wide_flights_dest_month.csv
# Treat csv as a sql table and perform sql queries against it
csvsql --query "select avg(mpg) as mean_mpg from mtcars_001" dat/mtcars_001.csv
# Create table statement
csvsql -i oracle --table mtcars dat/mtcars_001.csv
# Would like to see: way of outputting the individual `insert' statements.
#### csvcut
# Just show columns and column indexes
csvcut -n dat/mtcars_001.csv
# Look at columns 1 and 11
csvcut -c 1,11 dat/mtcars_001.csv | csvlook
# Look at columns by name
csvcut -c mpg,wt dat/mtcars_001.csv | csvlook
# Add line numbers
csvcut -c mpg,wt -l dat/mtcars_001.csv | csvlook
#### csvgrep
# Search for things with "azd" in name
csvgrep -c name -m "azd" dat/mtcars_001.csv | csvlook
# Search for things with 'a' as second letter from front
csvgrep -c name -r "^.a" dat/mtcars_001.csv | csvlook
#### csvsort
# Sort on first column
csvsort -c name dat/mtcars_001.csv | csvlook
# Sort first column reverse
csvsort -r -c name dat/mtcars_001.csv | csvlook
#### csvformat
# Output to tab delimited
csvformat -T dat/mtcars_001.csv
# Output to pipe delimited
csvformat -D\| dat/mtcars_001.csv
#### csvjson
# Convert
csvjson dat/mtcars_001.csv
# Convert, and indent
csvjson -i 4 dat/mtcars_001.csv | less
# Convert, indent, name the dicts
csvjson -i 4 -k name dat/mtcars_001.csv | less
#### csvpy
# open into python shell to manipulate
csvpy dat/mtcars_001.csv
#### csvjoin
# Inner, outer, left, and right joins
cat dat/join_1.csv
cat dat/join_2.csv
csvjoin -c "k" dat/join_1.csv dat/join_2.csv | csvlook
csvjoin -c "k" --outer dat/join_1.csv dat/join_2.csv | csvlook
csvjoin -c "k" --left dat/join_1.csv dat/join_2.csv | csvlook
csvjoin -c "k" --right dat/join_1.csv dat/join_2.csv | csvlook
#### Fancier interaction with bash
# Show headers for groups
for f in $(ls dat/mtcars_00*.csv); do
head -1 $f
done | csvlook -H
# someone missing a header?
for f in $(ls dat/mtcars_00*.csv dat/mtcars_noheader.csv | gshuf); do
head -1 $f
done | csvlook -H
# One file with headers out of order
for f in $(ls dat/mtcars_00*.csv dat/mtcars_col-order-shuffle.csv | gshuf); do
head -1 $f
done | csvlook -H
# Alternate, no csvkit tools
for f in $(ls dat/mtcars_00*.csv dat/mtcars_col-order-shuffle.csv | gshuf); do
head -1 $f
done | sort | uniq -c
# For larger set of files
for f in $(ls dat/mtcars_*.csv); do
head -1 $f
done | csvlook -H
# Alt, bash tools
for f in $(ls dat/mtcars_*.csv); do
head -1 $f
done | sort | uniq -c
# Visually browse the files
for f in $(ls dat/mtcars_00*.csv dat/mtcars_blank-select_cells.csv dat/mtcars_blank-out-row.csv dat/mtcars_noheader.csv | gshuf); do
echo
echo
echo "-----"
echo "file: " $f
echo
head $f | csvlook
done | less
# Visually browse some config files
for f in $(ls example_cfg/*.csv); do
echo
echo
echo "file: " $f
echo "=================================="
echo
head $f | csvlook
done | less -S