From 621f2ecded8606c46281401276139cae94af0c60 Mon Sep 17 00:00:00 2001 From: Jake Curragh Date: Sat, 8 Oct 2022 07:38:33 +0100 Subject: [PATCH 1/4] feat(terraform): add Terraform package --- terraform/README.md | 99 +++++++++++++++++++++++++++++++++++++++++++ terraform/install.ps1 | 57 +++++++++++++++++++++++++ terraform/install.sh | 32 ++++++++++++++ terraform/releases.js | 55 ++++++++++++++++++++++++ test/install.sh | 3 ++ 5 files changed, 246 insertions(+) create mode 100644 terraform/README.md create mode 100644 terraform/install.ps1 create mode 100644 terraform/install.sh create mode 100644 terraform/releases.js diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 000000000..1ebd83522 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,99 @@ +--- +title: Terraform +homepage: https://www.terraform.io/ +tagline: | + Provision, change, and version resources on any environment. +description: | + Terraform is an infrastructure as code (IaC) tool that allows you to build, change, and version infrastructure safely and efficiently. +--- + +## Cheat Sheet + +> With HashiCorp Terraform, provisioning and security can be automated based on +> infrastructure and policy as code. Infrastructure and policies are codified, +> shared, managed, and executed within a workflow that is consistent across all +> infrastructure. + +### Defining infrastructure state + +Create configurations that provide an outline for Terraform to provision your +target infrastructure. E.g., + +```tf +# main.tf + +terraform { + required_providers { + docker = { + source = "kreuzwerker/docker" + version = "~> 2.13.0" + } + } +} + +provider "docker" { + # provider-specific configuration goes here, e.g., + # browse provider documentation here https://registry.terraform.io/browse/providers +} + +# Define some resources! +resource "docker_image" "nginx" { + name = "nginx:latest" + keep_locally = false +} + +resource "docker_container" "nginx" { + image = docker_image.nginx.latest + name = "web-server" + ports { + internal = 80 + external = 8080 + } +} +``` + +### Initializing Terraform + +Terraform needs to install provider-specific plugins, generate lockfiles, etc. +before you can begin provisioning. + +`terraform init` + +You should only need to run this on new configurations, or other configurations +checked-out from version control. + +### Checking your configuration + +To check you have a valid configuration + +`terraform validate` + +To format your configuration files + +`terraform fmt` + +### Provisioning resources + +You can generate an execution plan before commiting to provisioning real +resources. This command allows you to see exactly what Terraform will do when +running the next command. + +`terraform plan` + +**Then, to apply your configurations and provision infrastructure resources.** + +`terraform apply` + +To automatically accept all user prompts when running this command + +`terraform apply -auto-approve` + +### Execution plans + +Execution plans generated by `terraform plan` also act as the _last working +state of your infrastructure_. You may wish to save the generated `.tfstate` +file so that you may re-provision these resources reliably. + +You can pass in the execution plan to `terraform apply` (example): + +`terraform apply -auto-approve main.tf` diff --git a/terraform/install.ps1 b/terraform/install.ps1 new file mode 100644 index 000000000..98f84b96f --- /dev/null +++ b/terraform/install.ps1 @@ -0,0 +1,57 @@ +#!/usr/bin/env pwsh + +################# +# Install terraform # +################# + +# Every package should define these variables +$pkg_cmd_name = "terraform" + +$pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\terraform.exe" +$pkg_dst_bin = "$Env:USERPROFILE\.local\bin" +$pkg_dst = "$pkg_dst_cmd" + +$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\terraform-v$Env:WEBI_VERSION\bin\terraform.exe" +$pkg_src_bin = "$Env:USERPROFILE\.local\opt\terraform-v$Env:WEBI_VERSION\bin" +$pkg_src_dir = "$Env:USERPROFILE\.local\opt\terraform-v$Env:WEBI_VERSION" +$pkg_src = "$pkg_src_cmd" + +New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | out-null +$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE" + +# Fetch archive +IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) { + echo "Downloading terraform from $Env:WEBI_PKG_URL to $pkg_download" + & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part" + & move "$pkg_download.part" "$pkg_download" +} + +IF (!(Test-Path -Path "$pkg_src_cmd")) { + echo "Installing terraform" + + # TODO: create package-specific temp directory + # Enter tmp + pushd .local\tmp + + # Remove any leftover tmp cruft + Remove-Item -Path ".\terraform-v*" -Recurse -ErrorAction Ignore + Remove-Item -Path ".\terraform.exe" -Recurse -ErrorAction Ignore + + # Unpack archive file into this temporary directory + # Windows BSD-tar handles zip. Imagine that. + echo "Unpacking $pkg_download" + & tar xf "$pkg_download" + + # Settle unpacked archive into place + echo "Install Location: $pkg_src_cmd" + New-Item "$pkg_src_bin" -ItemType Directory -Force | out-null + Move-Item -Path ".\terraform.exe" -Destination "$pkg_src_bin" + + # Exit tmp + popd +} + +echo "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'" +Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore | out-null +New-Item "$pkg_dst_bin" -ItemType Directory -Force | out-null +Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse diff --git a/terraform/install.sh b/terraform/install.sh new file mode 100644 index 000000000..54e4e4f27 --- /dev/null +++ b/terraform/install.sh @@ -0,0 +1,32 @@ +#!/bin/sh +set -e +set -u + +pkg_cmd_name="terraform" + +WEBI_SINGLE=true + +pkg_get_current_version() { + terraform -v 2> /dev/null | + head -n 1 | + cut -d 'v' -f 2 +} + +pkg_install() { + # $HOME/.local/bin/opt/terraform-v1.3.2/bin + mkdir -p "$pkg_src_bin" + + # mv ./terraform* "$HOME/.local/opt/terraform-v1.3.2/bin/terraform" + mv ./"$pkg_cmd_name"* "$pkg_src_cmd" + + # chmod a+x "$HOME/.local/opt/terraform-v1.3.2/bin/terraform" + chmod a+x "$pkg_src_cmd" +} + +pkg_link() { + # rm -f "$HOME/.local/bin/terraform" + rm -f "$pkg_dst_cmd" + + # ln -s "$HOME/.local/opt/terraform-v1.3.2/bin/terraform" "$HOME/.local/bin/terraform" + ln -s "$pkg_src_cmd" "$pkg_dst_cmd" +} diff --git a/terraform/releases.js b/terraform/releases.js new file mode 100644 index 000000000..61e1cb1c1 --- /dev/null +++ b/terraform/releases.js @@ -0,0 +1,55 @@ +'use strict'; + +let convert = { + freebsd: 'freebsd', + macos: 'darwin', + linux: 'linux', + windows: 'windows', + amd64: 'amd64', + arm: 'arm64', + 386: 'x86', +}; + +function getAllReleases(request) { + return request({ + url: 'https://releases.hashicorp.com/terraform/index.json', + json: true, + }).then(function (resp) { + let releases = resp.body; + let all = { + releases: [], + download: '', // Full URI provided in response body + }; + + function getBuildsForVersion(version) { + releases.versions[version].builds.forEach(function (build) { + let r = { + version: build.version, + download: build.url, + os: convert[build.os], + arch: convert[build.arch], + channel: 'stable', // No other channels + }; + all.releases.push(r); + }); + } + + // Releases are listed chronologically, we want the latest first. + const allVersions = Object.keys(releases.versions).reverse(); + + allVersions.forEach(function (version) { + getBuildsForVersion(version); + }); + + return all; + }); +} + +module.exports = getAllReleases; + +if (module === require.main) { + getAllReleases(require('@root/request')).then(function (all) { + all = require('../_webi/normalize.js')(all); + console.info(JSON.stringify(all)); + }); +} diff --git a/test/install.sh b/test/install.sh index 19178693e..7e0b4bcb7 100644 --- a/test/install.sh +++ b/test/install.sh @@ -76,6 +76,7 @@ __rmrf_local() { ssh-pubkey \ ssh-utils \ syncthing \ + terraform \ vim-ale \ vim-essentials \ vim-lastplace \ @@ -171,6 +172,7 @@ __rmrf_local() { ssh-pubkey \ ssh-utils \ syncthing \ + terraform \ vim-ale \ vim-essentials \ vim-go \ @@ -271,6 +273,7 @@ __test() { ssh-pubkey \ ssh-utils \ syncthing \ + terraform \ vim-ale \ vim-essentials \ vim-lastplace \ From aa616be91772364e97c80a9bbcc94783df9536e7 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Mon, 16 Oct 2023 19:56:45 +0000 Subject: [PATCH 2/4] ref(terraform): oppa AJ style --- terraform/README.md | 65 ++++++++++++++++++++++++++++++++------------ terraform/install.sh | 16 +++++++---- 2 files changed, 58 insertions(+), 23 deletions(-) diff --git a/terraform/README.md b/terraform/README.md index 1ebd83522..8eac2f608 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -7,6 +7,20 @@ description: | Terraform is an infrastructure as code (IaC) tool that allows you to build, change, and version infrastructure safely and efficiently. --- +To update or switch versions, run `webi terraform@stable` (or `@v1.6.1`, +`@beta`, etc). + +### Files + +These are the files / directories that are created and/or modified with this +install: + +```text +~/.config/envman/PATH.env +~/.local/bin/terraform +/main.tf +``` + ## Cheat Sheet > With HashiCorp Terraform, provisioning and security can be automated based on @@ -14,14 +28,14 @@ description: | > shared, managed, and executed within a workflow that is consistent across all > infrastructure. -### Defining infrastructure state +### How to Define Infrastructure State Create configurations that provide an outline for Terraform to provision your -target infrastructure. E.g., +target infrastructure. For example: -```tf -# main.tf +`main.tf`: +```tf terraform { required_providers { docker = { @@ -52,43 +66,56 @@ resource "docker_container" "nginx" { } ``` -### Initializing Terraform +### How to Initialize Terraform Terraform needs to install provider-specific plugins, generate lockfiles, etc. before you can begin provisioning. -`terraform init` +```sh +terraform init +``` You should only need to run this on new configurations, or other configurations checked-out from version control. -### Checking your configuration +### How to Lint / Check / Validate your Config To check you have a valid configuration -`terraform validate` +```sh +terraform validate +``` To format your configuration files -`terraform fmt` +```sh +terraform fmt +``` -### Provisioning resources +### How to Provision Resources -You can generate an execution plan before commiting to provisioning real +You can generate an execution plan before committing to provisioning real resources. This command allows you to see exactly what Terraform will do when running the next command. -`terraform plan` +```sh +terraform plan +``` -**Then, to apply your configurations and provision infrastructure resources.** +Then, **to apply your configurations and provision infrastructure** resources: -`terraform apply` +```sh +terraform apply +``` -To automatically accept all user prompts when running this command +Use `-auto-approve` to automatically accept all user prompts (non-interactive, +batch mode): -`terraform apply -auto-approve` +```sh +terraform apply -auto-approve +``` -### Execution plans +### How to Execute Plans Execution plans generated by `terraform plan` also act as the _last working state of your infrastructure_. You may wish to save the generated `.tfstate` @@ -96,4 +123,6 @@ file so that you may re-provision these resources reliably. You can pass in the execution plan to `terraform apply` (example): -`terraform apply -auto-approve main.tf` +```sh +terraform apply -auto-approve ./main.tf +``` diff --git a/terraform/install.sh b/terraform/install.sh index 54e4e4f27..4a68d6b0f 100644 --- a/terraform/install.sh +++ b/terraform/install.sh @@ -7,19 +7,25 @@ pkg_cmd_name="terraform" WEBI_SINGLE=true pkg_get_current_version() { + # 'terraform -v' has output in this format: + # Terraform v1.6.1 + # on linux_amd64 + # This trims it down to just the version number: + # 1.6.1 terraform -v 2> /dev/null | head -n 1 | - cut -d 'v' -f 2 + cut -d' ' -f2 | + cut -c2- } pkg_install() { - # $HOME/.local/bin/opt/terraform-v1.3.2/bin + # $HOME/.local/bin/opt/terraform-v1.6.1/bin mkdir -p "$pkg_src_bin" - # mv ./terraform* "$HOME/.local/opt/terraform-v1.3.2/bin/terraform" + # mv ./terraform* "$HOME/.local/opt/terraform-v1.6.1/bin/terraform" mv ./"$pkg_cmd_name"* "$pkg_src_cmd" - # chmod a+x "$HOME/.local/opt/terraform-v1.3.2/bin/terraform" + # chmod a+x "$HOME/.local/opt/terraform-v1.6.1/bin/terraform" chmod a+x "$pkg_src_cmd" } @@ -27,6 +33,6 @@ pkg_link() { # rm -f "$HOME/.local/bin/terraform" rm -f "$pkg_dst_cmd" - # ln -s "$HOME/.local/opt/terraform-v1.3.2/bin/terraform" "$HOME/.local/bin/terraform" + # ln -s "$HOME/.local/opt/terraform-v1.6.1/bin/terraform" "$HOME/.local/bin/terraform" ln -s "$pkg_src_cmd" "$pkg_dst_cmd" } From b138dd5ef298b648a9947f649925fc9473fd8cb0 Mon Sep 17 00:00:00 2001 From: Don Johnson Date: Thu, 12 Oct 2023 14:51:38 -0700 Subject: [PATCH 3/4] feat: add xsv (query CSV like SQL) --- xsv/README.md | 58 +++++++++++++++++++++++++++++++++++++++++++++ xsv/install.ps1 | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ xsv/install.sh | 46 ++++++++++++++++++++++++++++++++++++ xsv/releases.js | 20 ++++++++++++++++ 4 files changed, 186 insertions(+) create mode 100644 xsv/README.md create mode 100644 xsv/install.ps1 create mode 100644 xsv/install.sh create mode 100644 xsv/releases.js diff --git a/xsv/README.md b/xsv/README.md new file mode 100644 index 000000000..a680232ea --- /dev/null +++ b/xsv/README.md @@ -0,0 +1,58 @@ +--- +title: xsv +homepage: https://github.com/BurntSushi/xsv +tagline: | + xsv: A fast CSV command line toolkit written in Rust. +--- + +To update or switch versions, run `webi xsv@stable` (or `@v2`, `@beta`, etc). + +### Files + +These are the files / directories that are created and/or modified with this +install: + +```text +~/.config/envman/PATH.env +~/.local/bin/xsv +~/.local/opt/xsv +``` + +## Cheat Sheet + +> `xsv` is a command line program for manipulating CSV files. It offers a range +> of functionalities including slicing, joining, indexing and more. Designed for +> simplicity and speed, it is an essential tool for anyone working with CSV +> data. + +### Basic Usage + +To count the number of rows in a CSV file: + +```sh +xsv count data.csv +``` + +### Joining Two CSV Files + +To perform an inner join on two CSV files based on a common column: + +```sh +xsv join column1 file1.csv column2 file2.csv +``` + +### Sample Data + +To randomly sample rows from a CSV file: + +```sh +xsv sample 100 data.csv +``` + +### Analyzing Data + +To display basic statistics for each column in a CSV file: + +```sh +xsv stats data.csv +``` diff --git a/xsv/install.ps1 b/xsv/install.ps1 new file mode 100644 index 000000000..94141d993 --- /dev/null +++ b/xsv/install.ps1 @@ -0,0 +1,62 @@ +#!/usr/bin/env pwsh + +############### +# Install xsv # +############### + +# Every package should define these variables +$pkg_cmd_name = "xsv" + +$pkg_dst_cmd = "$Env:USERPROFILE\.local\bin\xsv.exe" +$pkg_dst_bin = "$Env:USERPROFILE\.local\bin" +$pkg_dst = "$pkg_dst_cmd" + +$pkg_src_cmd = "$Env:USERPROFILE\.local\opt\xsv-v$Env:WEBI_VERSION\bin\xsv.exe" +$pkg_src_bin = "$Env:USERPROFILE\.local\opt\xsv-v$Env:WEBI_VERSION\bin" +$pkg_src_dir = "$Env:USERPROFILE\.local\opt\xsv-v$Env:WEBI_VERSION" +$pkg_src = "$pkg_src_cmd" + +New-Item "$Env:USERPROFILE\Downloads\webi" -ItemType Directory -Force | out-null +$pkg_download = "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE" + +# Fetch archive +IF (!(Test-Path -Path "$Env:USERPROFILE\Downloads\webi\$Env:WEBI_PKG_FILE")) +{ + echo "Checking for (or Installing) MSVC Runtime..." + & "$Env:USERPROFILE\.local\bin\webi-pwsh.ps1" vcruntime + + echo "Downloading xsv from $Env:WEBI_PKG_URL to $pkg_download" + & curl.exe -A "$Env:WEBI_UA" -fsSL "$Env:WEBI_PKG_URL" -o "$pkg_download.part" + & move "$pkg_download.part" "$pkg_download" +} + +IF (!(Test-Path -Path "$pkg_src_cmd")) +{ + echo "Installing xsv" + + # TODO: create package-specific temp directory + # Enter tmp + pushd .local\tmp + + # Remove any leftover tmp cruft + Remove-Item -Path ".\xsv-v*" -Recurse -ErrorAction Ignore + Remove-Item -Path ".\xsv.exe" -Recurse -ErrorAction Ignore + + # Unpack archive file into this temporary directory + # Windows BSD-tar handles zip. Imagine that. + echo "Unpacking $pkg_download" + & tar xf "$pkg_download" + + # Settle unpacked archive into place + echo "Install Location: $pkg_src_cmd" + New-Item "$pkg_src_bin" -ItemType Directory -Force | out-null + Move-Item -Path ".\xsv.exe" -Destination "$pkg_src_bin" + + # Exit tmp + popd +} + +echo "Copying into '$pkg_dst_cmd' from '$pkg_src_cmd'" +Remove-Item -Path "$pkg_dst_cmd" -Recurse -ErrorAction Ignore | out-null +New-Item "$pkg_dst_bin" -ItemType Directory -Force | out-null +Copy-Item -Path "$pkg_src" -Destination "$pkg_dst" -Recurse diff --git a/xsv/install.sh b/xsv/install.sh new file mode 100644 index 000000000..3eaefd2be --- /dev/null +++ b/xsv/install.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# shellcheck disable=SC2034 +# "'pkg_cmd_name' appears unused. Verify it or export it." + +__init_xsv() { + set -e + set -u + + ################## + # Install xsv # + ################## + + # Every package should define these 6 variables + pkg_cmd_name="xsv" + + pkg_dst_cmd="$HOME/.local/bin/xsv" + pkg_dst="$pkg_dst_cmd" + + pkg_src_cmd="$HOME/.local/opt/xsv-v$WEBI_VERSION/bin/xsv" + pkg_src_dir="$HOME/.local/opt/xsv-v$WEBI_VERSION" + pkg_src="$pkg_src_cmd" + + # pkg_install must be defined by every package + pkg_install() { + # ~/.local/opt/xsv-v0.99.9/bin + mkdir -p "$(dirname "${pkg_src_cmd}")" + + # mv ./xsv-*/xsv ~/.local/opt/xsv-v0.99.9/bin/xsv + mv ./xsv "${pkg_src_cmd}" + } + + # pkg_get_current_version is recommended, but not required + pkg_get_current_version() { + # 'xsv --version' has output in this format: + # xsv 0.99.9 (rev abcdef0123) + # This trims it down to just the version number: + # 0.99.9 + xsv --version 2> /dev/null | + head -n 1 | + cut -d ' ' -f 2 + } + +} + +__init_xsv diff --git a/xsv/releases.js b/xsv/releases.js new file mode 100644 index 000000000..04cefc122 --- /dev/null +++ b/xsv/releases.js @@ -0,0 +1,20 @@ +'use strict'; + +var github = require('../_common/github.js'); +var owner = 'BurntSushi'; +var repo = 'xsv'; + +module.exports = function (request) { + return github(request, owner, repo).then(function (all) { + return all; + }); +}; + +if (module === require.main) { + module.exports(require('@root/request')).then(function (all) { + all = require('../_webi/normalize.js')(all); + // just select the first 5 for demonstration + all.releases = all.releases.slice(0, 5); + console.info(JSON.stringify(all, null, 2)); + }); +} From 8d4c099bc7d2098d58fef1a6c6a0fc04c0f1db30 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Mon, 16 Oct 2023 23:07:55 +0000 Subject: [PATCH 4/4] doc(xsv): reorganized and updated with examples --- xsv/README.md | 577 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 564 insertions(+), 13 deletions(-) diff --git a/xsv/README.md b/xsv/README.md index a680232ea..59ffd01f6 100644 --- a/xsv/README.md +++ b/xsv/README.md @@ -5,7 +5,7 @@ tagline: | xsv: A fast CSV command line toolkit written in Rust. --- -To update or switch versions, run `webi xsv@stable` (or `@v2`, `@beta`, etc). +To update or switch versions, run `webi xsv@stable` (or `@v0.13`, `@beta`, etc). ### Files @@ -15,7 +15,6 @@ install: ```text ~/.config/envman/PATH.env ~/.local/bin/xsv -~/.local/opt/xsv ``` ## Cheat Sheet @@ -25,34 +24,586 @@ install: > simplicity and speed, it is an essential tool for anyone working with CSV > data. +Get the canonical sample data: + +```sh +curl -o ./worldcitiespop.csv -L https://burntsushi.net/stuff/worldcitiespop.csv +``` + +(or check "The Data Science Toolkit": ) + +Show the CSV's headers: + +```sh +xsv headers ./worldcitiespop.csv +``` + +Query the CSV data any which way: + +```sh +xsv search '^(John|Jane)$' --select 'First Name' ./address-book.csv | + xsv select 'ID,First Name,Last Name' | + xsv sort --select 'Last Name,First Name' | + xsv slice -s 0 -n 5 + xsv table +``` + +(selects the first 5 rows, with a "First Name" of "John" or "Jane", sorted by +"Last Name", as a table) + ### Basic Usage -To count the number of rows in a CSV file: +- Subcommand Help & Global Options +- View Headers +- Take a Sample +- Select Columns +- View Vertically (`\G`) +- View as Table +- Count Rows + +**Subcommand Help & Global Options** ```sh -xsv count data.csv +xsv --help +``` + +Subcommands are: `cat`, `count`, `fixlengths`, `flatten`, `fmt`, `frequency`, +`headers`, `help`, `index`, `input`, `join`, `sample`, `search`, `select`, +`slice`, `sort`, `split`, `stats`, `table`. + +```text +Common options: + -h, --help Display this message + -o, --output Write output to instead of stdout. + -n, --no-headers When set, the first row will not be interpreted + as headers. (i.e., They are not searched, analyzed, + sliced, etc.) + -d, --delimiter The field delimiter for reading CSV data. + Must be a single character. (default: ,) ``` -### Joining Two CSV Files +**View Headers** -To perform an inner join on two CSV files based on a common column: +Shows column name and index (1-based). ```sh -xsv join column1 file1.csv column2 file2.csv +xsv headers ./worldcitiespop.csv ``` -### Sample Data +```text +1 Country +2 City +3 AccentCity +4 Region +5 Population +6 Latitude +7 Longitude +``` -To randomly sample rows from a CSV file: +**Sample Data** ```sh -xsv sample 100 data.csv +xsv sample 2 ./worldcitiespop.csv ``` -### Analyzing Data +```text +Country,City,AccentCity,Region,Population,Latitude,Longitude +us,provo,Provo,UT,105764,40.2338889,-111.6577778 +lv,riga,Riga,25,742570,56.95,24.1 +``` -To display basic statistics for each column in a CSV file: +**Select Columns** ```sh -xsv stats data.csv +xsv sample 2 ./worldcitiespop.csv | + xsv select Country,City +``` + +```text +Country,City +us,provo +lv,riga +``` + +```sh +xsv sample 2 ./worldcitiespop.csv | + xsv select --no-headers 1,2 +``` + +```text +us,provo +lv,riga +``` + +**Select Rows** + +Rows are **0-indexed** and do not include the header, unless `--no-headers` is +given. + +```sh +# xsv slice -s -l +xsv slice -s 0 -l 2 ./worldcitiespop.csv | + xsv table +``` + +```text +Country City AccentCity Region Population Latitude Longitude +ad aixas Aixàs 06 42.4833333 1.4666667 +ad aixirivali Aixirivali 06 42.4666667 1.5 +``` + +**View Vertically** + +Like `\G` in SQL. + +```sh +xsv sample 2 ./worldcitiespop.csv | + xsv select Country,City | + xsv flatten +``` + +```text +Country us +City provo +# +Country lv +City riga +``` + +**View as a Table** + +Makes all columns the same width, truncated to `-c N`. + +```sh +xsv search 'Provo|Riga' ./worldcitiespop.csv | + xsv table -c 10 +``` + +```text +Country City AccentCity Region Population Latitude Longitude +us provo Provo UT 105764 40.2338889 -111.65777... +lv riga Riga 25 742570 56.95 24.1 +``` + +### How to Query & Analyze Data + +**Count Number of Rows**: + +```sh +xsv count ./worldcitiespop.csv +``` + +```text +3173958 +``` + +**Search by Column Value** + +```sh +# xsv search [--select 'Column 1,Column 2'] +xsv search '^(provo|riga)$' --select 'City' ./worldcitiespop.csv | + xsv search '^(us|lv)$' --select Country +``` + +**Join CSVs** + +Like a SQL INNER JOIN (other options available). + +```sh +# The equivalent of +# SELECT * +# FROM "countries-by-name" +# INNER JOIN "countries-by-code" +# ON "Country Code" = "Code" + +xsv join \ + "Country Code" ./contries-by-name.csv \ + "Code" countries-by-code.csv +``` + +**Basic Statistics** + +```sh +xsv stats ./worldcitiespop.csv | + xsv table -c 11 +``` + +```text +field type sum min max min_length max_length mean stddev +Country Unicode ad zw 2 2 +City Unicode bab el ahm... Þykkvibaer 1 91 +AccentCity Unicode Bâb el Ahm... ïn Bou Chel... 1 91 +Region Unicode 00 Z9 0 2 +Population Integer 2289584999 7 31480498 0 8 47719.57063... 302885.5592... +Latitude Float 86294096.37... -54.933333 82.483333 1 12 27.18816580... 21.95261384... +Longitude Float 117718483.5... -179.983333... 180 1 14 37.08885989... 63.22301045... +``` + +### Extended Help + +We don't generally print the full help, but since this is so vast and it's +useful to be able to search it all on a single page... + +```text +cat Concatenate by row or column +count Count records +fixlengths Makes all records have same length +flatten Show one field per line +fmt Format CSV output (change field delimiter) +frequency Show frequency tables +headers Show header names +index Create CSV index for faster access +input Read CSV data with special quoting rules +join Join CSV files +sample Randomly sample CSV data +search Search CSV data with regexes +select Select columns from CSV +slice Slice records from CSV +sort Sort CSV data +split Split CSV data into many files +stats Compute basic statistics +table Align CSV data into columns +``` + +**How to view ALL help, at once** + +```sh +xsv --list | + tail -n +2 | + grep -v '^$' | + cut -c 5- | + cut -d' ' -f1 | + xargs -I '{}' xsv '{}' --help +``` + +**All Usage and Options** + +(descriptions and common help omitted) + +#### `xsv cat --help` + +```text +Usage: + xsv cat rows [options] [...] + xsv cat columns [options] [...] + xsv cat --help + +cat options: + -p, --pad When concatenating columns, this flag will cause + all records to appear. It will pad each row if + other CSV data isn't long enough. +``` + +#### `xsv count --help` + +```text +Usage: + xsv count [options] [] +``` + +#### `xsv fixlengths --help` + +```text +Usage: + xsv fixlengths [options] [] + +fixlengths options: + -l, --length Forcefully set the length of each record. If a + record is not the size given, then it is truncated + or expanded as appropriate. +``` + +#### `xsv flatten --help` + +```text +Usage: + xsv flatten [options] [] + +flatten options: + -c, --condense Limits the length of each field to the value + specified. If the field is UTF-8 encoded, then + refers to the number of code points. + Otherwise, it refers to the number of bytes. + -s, --separator A string of characters to write after each record. + When non-empty, a new line is automatically + appended to the separator. + [default: #] +``` + +#### `xsv fmt --help` + +```text +Usage: + xsv fmt [options] [] + +fmt options: + -t, --out-delimiter The field delimiter for writing CSV data. + [default: ,] + --crlf Use '\r\n' line endings in the output. + --ascii Use ASCII field and record separators. + --quote The quote character to use. [default: "] + --quote-always Put quotes around every value. + --escape The escape character to use. When not specified, + quotes are escaped by doubling them. +``` + +#### `xsv frequency --help` + +```text +Usage: + xsv frequency [options] [] + +frequency options: + -s, --select Select a subset of columns to compute frequencies + for. See 'xsv select --help' for the format + details. This is provided here because piping 'xsv + select' into 'xsv frequency' will disable the use + of indexing. + -l, --limit Limit the frequency table to the N most common + items. Set to '0' to disable a limit. + [default: 10] + -a, --asc Sort the frequency tables in ascending order by + count. The default is descending order. + --no-nulls Don't include NULLs in the frequency table. + -j, --jobs The number of jobs to run in parallel. + This works better when the given CSV data has + an index already created. Note that a file handle + is opened for each job. + When set to '0', the number of jobs is set to the + number of CPUs detected. + [default: 0] +``` + +#### `xsv headers --help` + +```text +Usage: + xsv headers [options] [...] + +headers options: + -j, --just-names Only show the header names (hide column index). + This is automatically enabled if more than one + input is given. + --intersect Shows the intersection of all headers in all of + the inputs given. +``` + +#### `xsv index --help` + +```text +Usage: + xsv index [options] + xsv index --help + +index options: + -o, --output Write index to instead of .idx. + Generally, this is not currently useful because + the only way to use an index is if it is specially + named .idx. +``` + +#### `xsv input --help` + +```text +Usage: + xsv input [options] [] + +input options: + --quote The quote character to use. [default: "] + --escape The escape character to use. When not specified, + quotes are escaped by doubling them. + --no-quoting Disable quoting completely. +``` + +#### `xsv join --help` + +```text +Usage: + xsv join [options] + xsv join --help + +join options: + --no-case When set, joins are done case insensitively. + --left Do a 'left outer' join. This returns all rows in + first CSV data set, including rows with no + corresponding row in the second data set. When no + corresponding row exists, it is padded out with + empty fields. + --right Do a 'right outer' join. This returns all rows in + second CSV data set, including rows with no + corresponding row in the first data set. When no + corresponding row exists, it is padded out with + empty fields. (This is the reverse of 'outer left'.) + --full Do a 'full outer' join. This returns all rows in + both data sets with matching records joined. If + there is no match, the missing side will be padded + out with empty fields. (This is the combination of + 'outer left' and 'outer right'.) + --cross USE WITH CAUTION. + This returns the cartesian product of the CSV + data sets given. The number of rows return is + equal to N * M, where N and M correspond to the + number of rows in the given data sets, respectively. + --nulls When set, joins will work on empty fields. + Otherwise, empty fields are completely ignored. + (In fact, any row that has an empty field in the + key specified is ignored.) +``` + +#### `xsv sample --help` + +```text +Usage: + xsv sample [options] [] + xsv sample --help + +Common options: + -h, --help Display this message + -o, --output Write output to instead of stdout. + -n, --no-headers When set, the first row will be consider as part of + the population to sample from. (When not set, the + first row is the header row and will always appear + in the output.) + -d, --delimiter The field delimiter for reading CSV data. + Must be a single character. (default: ,) +``` + +#### `xsv search --help` + +```text +Usage: + xsv search [options] [] + xsv search --help + +search options: + -i, --ignore-case Case insensitive search. This is equivalent to + prefixing the regex with '(?i)'. + -s, --select Select the columns to search. See 'xsv select -h' + for the full syntax. + -v, --invert-match Select only rows that did not match +``` + +#### `xsv select --help` + +```text + Select the first and fourth columns: + $ xsv select 1,4 + + Select the first 4 columns (by index and by name): + $ xsv select 1-4 + $ xsv select Header1-Header4 + + Ignore the first 2 columns (by range and by omission): + $ xsv select 3- + $ xsv select '!1-2' + + Select the third column named 'Foo': + $ xsv select 'Foo[2]' + + Re-order and duplicate columns arbitrarily: + $ xsv select 3-1,Header3-Header1,Header1,Foo[2],Header1 + + Quote column names that conflict with selector syntax: + $ xsv select '"Date - Opening","Date - Actual Closing"' + +Usage: + xsv select [options] [--] [] + xsv select --help +``` + +#### `xsv slice --help` + +```text +Usage: + xsv slice [options] [] + +slice options: + -s, --start The index of the record to slice from. + -e, --end The index of the record to slice to. + -l, --len The length of the slice (can be used instead + of --end). + -i, --index Slice a single record (shortcut for -s N -l 1). +``` + +#### `xsv sort --help` + +``` +Usage: + xsv sort [options] [] + +sort options: + -s, --select Select a subset of columns to sort. + See 'xsv select --help' for the format details. + -N, --numeric Compare according to string numerical value + -R, --reverse Reverse order +``` + +#### `xsv split --help` + +```text +Usage: + xsv split [options] [] + xsv split --help + +split options: + -s, --size The number of records to write into each chunk. + [default: 500] + -j, --jobs The number of spliting jobs to run in parallel. + This only works when the given CSV data has + an index already created. Note that a file handle + is opened for each job. + When set to '0', the number of jobs is set to the + number of CPUs detected. + [default: 0] + --filename A filename template to use when constructing + the names of the output files. The string '{}' + will be replaced by a value based on the value + of the field, but sanitized for shell safety. + [default: {}.csv] +``` + +#### `xsv stats --help` + +``` +Usage: + xsv stats [options] [] + +stats options: + -s, --select Select a subset of columns to compute stats for. + See 'xsv select --help' for the format details. + This is provided here because piping 'xsv select' + into 'xsv stats' will disable the use of indexing. + --everything Show all statistics available. + --mode Show the mode. + This requires storing all CSV data in memory. + --cardinality Show the cardinality. + This requires storing all CSV data in memory. + --median Show the median. + This requires storing all CSV data in memory. + --nulls Include NULLs in the population size for computing + mean and standard deviation. + -j, --jobs The number of jobs to run in parallel. + This works better when the given CSV data has + an index already created. Note that a file handle + is opened for each job. + When set to '0', the number of jobs is set to the + number of CPUs detected. + [default: 0] +``` + +#### `xsv table --help` + +```text +Usage: + xsv table [options] [] + +table options: + -w, --width The minimum width of each column. + [default: 2] + -p, --pad The minimum number of spaces between each column. + [default: 2] + -c, --condense Limits the length of each field to the value + specified. If the field is UTF-8 encoded, then + refers to the number of code points. + Otherwise, it refers to the number of bytes. ```