curl.functions

# This file is part of shellfire curl. It is subject to the licence terms in the COPYRIGHT file found in the top-level directory of this distribution and at https://raw.githubusercontent.com/shellfire-dev/curl/master/COPYRIGHT. No part of shellfire curl, including this file, may be copied, modified, propagated, or distributed except according to the terms contained in the COPYRIGHT file.
# Copyright © 2014-2015 The developers of shellfire curl. See the COPYRIGHT file in the top-level directory of this distribution and at https://raw.githubusercontent.com/shellfire-dev/curl/master/COPYRIGHT.


core_usesIn curl
core_usesIn urlencode

# Initialised Parameters:-
#  * curl_userAgent
#  * curl_maxiumRedirects
#  * curl_retries
# Out-of-band parameters:-
#  * curl_
# By verb parameters:-
#  HEAD, GET
#  * curl_GET_range (optional)
#  POST
#  * curl_POST (mandatory)

core_dependency_requires '*' curl cat
curl_http()
{
	# Use 'none' for none
	local authentication="$1"
	
	# Use 'user:password' for Basic Auth.  Specify just 'user' for password prompting with Basic Auth. Use 'token:x-oauth-basic' for OAuth authentication. Use ':' for NTLM (--ntlm) or Kerberos5 (--negotiate).
	# Use '' for no authentication, or to use authentication via .netrc
	local credentials="$2"
	
	local verb="$3"
	
	local url="$4"
	
	# outputFilePath may be '-' for stdout and '/dev/null' if unwanted.
	local outputFilePath="$5"
	
	shift 5
	
	if core_variable_isUnset _curl_initialised; then
		_curl_initialise 'shellfire'
	fi
	
	# Defensive measure to ensure that it is always set, even if curl fails catastrophically (eg exit code 51, SSL certificate mismatch), with a value that can never be a HTTP exit code
	curl_httpStatusCode=-1
	
	local curlOptions
	local curlOptions_initialised
	core_variable_array_initialise curlOptions
	
	# Prevent default configuration from being loaded
	core_variable_array_append curlOptions -q
	
	core_variable_array_append curlOptions --user-agent "$curl_userAgent"
	
	if core_variable_isTrue "$curl_supportsRetries"; then
		core_variable_array_append curlOptions --retry "$curl_retries"
		core_variable_array_append curlOptions --retry-delay "$curl_retryDelay"
		core_variable_array_append curlOptions --retry-max-time "$curl_retryMaximumDelay"
	fi
	
	core_variable_array_append curlOptions --compressed
	
	if core_variable_isTrue "$curl_supportsNetrc"; then
		
		local _curl_changeNetrcLocation_curlrcFile=''
		_curl_changeNetrcLocation _program_etcPath "$core_libraryName"/netrc
		_curl_changeNetrcLocation _program_etcPath "$_program_namespace"/netrc
		_curl_changeNetrcLocation CURL_HOME .netrc
		_curl_changeNetrcLocation HOME .netrc
		_curl_changeNetrcLocation HOME ."$core_libraryName".netrc
		_curl_changeNetrcLocation HOME ."$_program_name".netrc
		if [ -n "$_curl_changeNetrcLocation_curlrcFile" ]; then
			core_variable_array_append curlOptions --netrc-file "$_curl_changeNetrcLocation_curlrcFile"
		fi
		
	elif core_path_isReadableNonEmptyFilePath "$HOME"/.netrc; then
		
		core_variable_array_append curlOptions --netrc
		
	fi

	_curl_appendConfigFileIfExtant _program_etcPath "$core_libraryName"/curlrc
	_curl_appendConfigFileIfExtant _program_etcPath "$_program_namespace"/curlrc
	_curl_appendConfigFileIfExtant CURL_HOME .curlrc
	_curl_appendConfigFileIfExtant HOME .curlrc
	_curl_appendConfigFileIfExtant HOME ."$core_libraryName".curlrc
	_curl_appendConfigFileIfExtant HOME ."$_program_name".curlrc
	
	# Script-friendly output
	core_variable_array_append curlOptions --silent --show-error
	
	core_variable_array_append curlOptions --location
	
	core_variable_array_append curlOptions --location-trusted
	
	core_variable_array_append curlOptions --max-redirs "$curl_maximumRedirects"
	
	core_variable_array_append curlOptions --tlsv1
	
	if core_variable_isTrue "$curl_makeTlsInsecure"; then
		core_variable_array_append curlOptions --insecure
	fi
	
	# Uses HTTP Date header if available to set the timestamp of any downloaded file
	core_variable_array_append curlOptions --remote-time

	core_variable_array_append curlOptions --request "$verb"
	
	case "$verb" in
		
		HEAD|GET)
			:
		;;
		
		POST)
			core_variable_array_append curlOptions --data-binary @"$curl_uploadFile"
		;;
		
		PATCH)
			core_variable_array_append curlOptions --data-binary @"$curl_uploadFile"
		;;
		
		PUT)
			core_variable_array_append curlOptions --upload-file "$curl_uploadFile"
		;;
		
		DELETE)
			core_variable_array_append curlOptions --request DELETE
		;;
		
		*)
			core_exitError $core_commandLine_exitCode_SOFTWARE "Curl verb '$verb' is not supported."
		;;
		
	esac
	
	# Range is valid for any HTTP verb, although it is used almost exclusively with GET
	if core_variable_isSet curl_range; then
		if [ -n "$curl_range" ]; then
			core_variable_array_append curlOptions --range "$curl_range"
		fi
	fi
	
	# Continue-At is valid for any HTTP verb, although it is used mostly with PUT and less commonly with POST and PATCH
	if core_variable_isSet curl_continueAt; then
		if [ -n "$curl_continueAt" ]; then
			core_variable_array_append curlOptions --continue-at "$curl_continueAt"
		fi
	fi

	# Netscape cookie file format
	if core_variable_isSet curl_cookieJar; then
		if [ -n "$curl_cookieJar" ]; then
			core_variable_array_append curlOptions --cookie-jar "$curl_cookieJar"
		fi
	fi
	
	core_variable_array_append curlOptions --referer ';auto'
	
	core_variable_array_append curlOptions --output "$outputFilePath"
	
	core_variable_array_append curlOptions --dump-header "$curl_outputHeadersFilePath"
	
	if [ "$(core_init_verbosity)" -gt 1 ]; then
		core_variable_array_append curlOptions --stderr "$curl_stderrFilePath"
		core_variable_array_append curlOptions --write-out '%{http_code}\t%{url_effective}\t%{time_total}\t%{size_download}\t%{speed_download}\t%{num_redirects}\n'
	
		if [ "$(core_init_verbosity)" -gt 2 ]; then
			core_variable_array_append curlOptions --trace-time --trace-ascii "$curl_traceAsciiFilePath"
		fi
	fi
	
	# We pass credentials, URLs and Headers via a temporary, user-only config file so that sensitive information (eg auth tokens in query strings) aren't exposed
	# Of course, nothing stops one used .netrc and/or other config files to do this, too
	{
		_curl_writeConfigValue url "$url"
	
		case "$authentication" in
			
			none)
				if [ -n "$credentials" ]; then
					core_exitError $core_commandLine_exitCode_SOFTWARE "Don't specify credentials with authentication 'none'"
				fi
			;;
			
			# We do not support GSS-Level
			basic|digest|ntlm|negotiate)
				_curl_writeConfigValue "$authentication"
				_curl_writeConfigValue user "$credentials"
			;;
			
		esac
		
	
		local headerName
		local headerValue
		while [ $# -ne 0 ]
		do
			headerName="$1"
			headerValue="$2"
			shift 2
		
			if [ -z "$headerValue" ]; then
				_curl_writeConfigValue header "${headerName};"
			else
				_curl_writeConfigValue header "${headerName}: ${headerValue}"
			fi
		done
	}>"$curl_secureParametersFilePath"
	core_variable_array_append curlOptions --config "$curl_secureParametersFilePath"
	
	set +e
	core_variable_array_passToFunctionAsArguments curlOptions curl 1>"$curl_stdoutFilePath"
	# Worse than a 404 or even a 5xx; something terrible has happened (eg broken TCP connection after retries)
	local curl_exitCode=$?
	set -e
	
	# http://curl.haxx.se/libcurl/c/libcurl-errors.html
	if [ $curl_exitCode -ne 0 ]; then
		
		if core_variable_isFalse curl_failHard; then
			return $curl_exitCode
		fi
	
		case $curl_exitCode in
		
			# 'Never' events (22 should be impossible unless config has been messed up)
			1|2|3|4|22|27|41|42|43|48|53|54)
				core_exitError $core_commandLine_exitCode_SOFTWARE "Could not retrieve url '$url' using curl. Curl exit code was '$curl_exitCode' ('never' event)."
			;;
			
			8|10|11|13|14|16|17|18|19|25|30|31|34|47|49|51|52)
				core_exitError $core_commandLine_exitCode_PROTOCOL "Could not retrieve url '$url' using curl. Curl exit code was '$curl_exitCode' (protocol problems)."
			;;
			
			35|58)
				core_exitError $core_commandLine_exitCode_NOPERM "Could not retrieve url '$url' using curl. Curl exit code was '$curl_exitCode' (access denied)."
			;;
			
			# Transient
			5|6|7|9|12|15|21|28|45|55|56)
				core_exitError $core_commandLine_exitCode_UNAVAILABLE "Could not retrieve url '$url' using curl. Curl exit code was '$curl_exitCode' (transient issue)."
			;;
			
			# IO
			23|26|37)
				core_exitError $core_commandLine_exitCode_IOERR "Could not retrieve url '$url' using curl. Curl exit code was '$curl_exitCode' (IO problems)."
			;;
			
			# Might legitimately occur if probing a server
			33)
				core_exitError $core_commandLine_exitCode_PROTOCOL "Could not retrieve url '$url' using curl. Curl exit code was '$curl_exitCode' (Range Error)."
			;;
			
			# Might legitimately occur if trying to resume a download, and the resource has changed in the meantime
			36)
				core_exitError $core_commandLine_exitCode_PROTOCOL "Could not retrieve url '$url' using curl. Curl exit code was '$curl_exitCode' (Download Resume Error)."
			;;
			
			*)
				core_exitError $core_commandLine_exitCode_FAILURE "Could not retrieve url '$url' using curl. Curl exit code was '$curl_exitCode' (unknown cause)."
			;;
		esac
		
	fi
	
	if [ "$(core_init_verbosity)" -gt 1 ]; then
		{
			# "$(<FILE)" doesn't work in all situations
			core_message INFO "curl write-out: $(cat "$curl_stdoutFilePath")"
			core_message INFO "curl stderr (usually empty): $(cat "$curl_stderrFilePath")"
	
			if [ "$(core_init_verbosity)" -gt 2 ]; then
				printf '%s\n' "ascii-trace:-"
				cat "$curl_traceAsciiFilePath"
				printf '\n'
			fi
		} 1>&2
	fi
	
	_curl_retrieve_terminatePreviousHeaderLineIfPending()
	{
		if [ $headerLinePending -eq 1 ]; then
			printf '\n'
			headerLinePending=0
		fi
	}
	
	# https://tools.ietf.org/html/rfc7230#section-3.2.4
	# Designed to handle 100 Continue, with multiple responses
	# Character encoding is a mess. Historically, we were supposed to use ISO-8859-1 (ie Latin1) with RFC 2047 encoding.
	local headerExpectation=0
	local headerLinePending=0
	{
		while IFS="$_curl_CR" read -r headerLine
		do
			case $headerExpectation in
			
				0)
					IFS=' ' read -r curl_httpVersion curl_httpStatusCode curl_httpReasonPhrase <<-EOF
						${headerLine}
					EOF
					# Resets headers
					printf '' >"$curl_parsedHeadersFilePath"
					headerExpectation=1
				;;
			
				1)
					# look for folded lines
					case "$(core_variable_firstCharacter "$headerLine")" in
						
						# End of headers (empty line)
						'')
							_curl_retrieve_terminatePreviousHeaderLineIfPending
							headerExpectation=0
						;;
						
						# obs-fold (continuation line, deprecated but still allowed)
						' '|"$_curl_HT")
							printf ' %s' "$headerLine"
						;;
					
						*)
							_curl_retrieve_terminatePreviousHeaderLineIfPending
							printf '%s' "$headerLine"
							headerLinePending=1
						;;
					
					esac
				;;
			
			esac
		
		done <"$curl_outputHeadersFilePath"
		
		_curl_retrieve_terminatePreviousHeaderLineIfPending
		
	} >"$curl_parsedHeadersFilePath"
}

_curl_writeConfigValue()
{
	# Values in double quotes have \escape sequences expanded; without double quotes, whitespaces aren't allowed. One can't win.
	if [ $# -eq 2 ]; then
		printf '%s = "%s"\n' "$1" "$2"
	else
		printf '%s\n' "$1"
	fi
}

core_dependency_requires '*' tr
curl_iterateHeaders()
{
	local mixedCaseHeaderName
	local headerName
	local headerValue
	local callback
	
	local headerField
	local fieldName
	local fieldValue
	while IFS='' read -r headerField
	do
		# Mixed case, arggh
		fieldName="${headerField%%:*}"
		fieldValue="$(core_variable_trimSpaceAndHorizontalTab "${headerField#*:}")"
		
		fieldName="$(printf '%s' "$fieldName" | tr 'a-z' 'A-Z')"
		for callback in "$@"
		do
			$callback
		done
	done <"$curl_parsedHeadersFilePath"
}

core_usesIn version
core_dependency_requires '*' curl awk
_curl_initialise()
{
	if core_variable_isUnset curl_userAgent; then
		curl_userAgent="$1"
	fi
	
	# Default is 50. -1 makes it limitless.
	if core_variable_isUnset curl_maximumRedirects; then
		curl_maximumRedirects=5
	fi
	
	if core_variable_isUnset curl_retries; then
		curl_retries=10
	fi
	
	if core_variable_isUnset curl_retryDelay; then
		curl_retryDelay=0
	fi
	
	if core_variable_isUnset curl_retryMaximumDelay; then
		curl_retryMaximumDelay=30
	fi

	if core_variable_isUnset curl_failHard; then
		curl_failHard=1
	fi

	_curl_CR="$(printf '\r')"
	_curl_HT="$(printf '\t')"
	_curl_headerIFS=": $_curl_HT"
	
	local TMP_FILE
	
	# Used to pass sensitive URLs and Headers (eg those containing passwords) that should not be leaked to the command-line (eg those containing sensitive tokens - GitHub's API is an example of this)
	if core_variable_isUnset curl_secureParametersFilePath; then
		core_temporaryFiles_newFileToRemoveOnExit
		curl_secureParametersFilePath="$TMP_FILE"
	fi

	if core_variable_isUnset curl_outputHeadersFilePath; then
		core_temporaryFiles_newFileToRemoveOnExit
		curl_outputHeadersFilePath="$TMP_FILE"
	fi
	
	if core_variable_isUnset curl_parsedHeadersFilePath; then
		core_temporaryFiles_newFileToRemoveOnExit
		curl_parsedHeadersFilePath="$TMP_FILE"
	fi
	
	if core_variable_isUnset curl_stdoutFilePath; then
		core_temporaryFiles_newFileToRemoveOnExit
		curl_stdoutFilePath="$TMP_FILE"
	fi
	
	if [ "$(core_init_verbosity)" -gt 1 ]; then
	
		if core_variable_isUnset curl_stderrFilePath; then
			core_temporaryFiles_newFileToRemoveOnExit
			curl_stderrFilePath="$TMP_FILE"
		fi
	
		if core_variable_isUnset curl_traceAsciiFilePath; then
			core_temporaryFiles_newFileToRemoveOnExit
			curl_traceAsciiFilePath="$TMP_FILE"
		fi
		
	fi
	
	# Curl versioning determines specific features
	# CentOS 5 and 6 are still in widespread use, and use archaic curl versions
	
	local curlVersion="$(curl --version | awk -F ' ' 'NR==1 {print $2}')"
	
	if core_variable_isUnset curl_supportsNetrc; then
		if version_isGreaterThanOrEqualTo "$curlVersion" 7.21.5; then
			curl_supportsNetrc='yes'
		else
			curl_supportsNetrc='no'
		fi
	fi
	
	# This is the exact version in CentOS 6
	# Versions in CentOS 5 seem to suffer from a problem with subjectAltName certs
	if core_variable_isUnset curl_makeTlsInsecure; then
		if version_isGreaterThanOrEqualTo "$curlVersion" 7.19.7; then
			curl_makeTlsInsecure='no'
		else
			core_message WARN "Making curl TLS connections insecure by default because you're running an antique version of curl. Override by setting curl_makeTlsInsecure=no"
			curl_makeTlsInsecure='yes'
		fi
	fi

	if core_variable_isUnset curl_supportsRetries; then
		if version_isGreaterThanOrEqualTo "$curlVersion" 7.12.3; then
			curl_supportsRetries='yes'
		else
			curl_supportsRetries='no'
		fi
	fi
	
	_curl_initialised='yes'
}

# Switch can be specified multiple times, last one wins
_curl_changeNetrcLocation()
{
	local variableNameOfPath="$1"
	local fileName="$2"
	
	if core_variable_isUnset "$variableNameOfPath"; then
		# This logic handles HOME being unset
		if [ "$variableNameOfPath" = 'HOME' ]; then
			if [ "$fileName" ='.curlc' ]; then
				if core_path_isReadableNonEmptyFilePath "$HOME"/.netrc; then
					_curl_changeNetrcLocation_curlrcFile="$HOME"/.netrc
				fi
			fi
		fi
		return 0
	fi
	
	local core_variable_indirectValue_result
	core_variable_indirectValue "$variableNameOfPath"
	
	local curlrcFile="$core_variable_indirectValue_result"/"$fileName"
	
	if core_path_isReadableNonEmptyFilePath "$curlrcFile"; then
		_curl_changeNetrcLocation_curlrcFile="$curlrcFile"
	fi
}

_curl_appendConfigFileIfExtant()
{
	local variableNameOfPath="$1"
	local fileName="$2"
	
	if core_variable_isUnset "$variableNameOfPath"; then
		# This logic handles HOME being unset
		if [ "$variableNameOfPath" = 'HOME' ]; then
			if [ "$fileName" ='.curlc' ]; then
				if core_path_isReadableNonEmptyFilePath "$HOME"/.curlrc; then
					core_variable_array_append --config "$HOME"/.curlrc
				fi
			fi
		fi
		return 0
	fi
	
	local core_variable_indirectValue_result
	core_variable_indirectValue "$variableNameOfPath"
	
	local curlrcFile="$core_variable_indirectValue_result"/"$fileName"
	
	if core_path_isReadableNonEmptyFilePath "$curlrcFile"; then
		core_variable_array_append --config "$curlrcFile"
	fi
}