jfrogGH-621 - working for groups, users and repos.

next steps: handle the special case of docker local repos (we need to fetch the version number which requires an extra call)
test management of netrc (we don't want to mess up someones file)
exclude resources found in the local terraform state

Dockerfile

@@ -1,23 +1,35 @@
-# Fetch the dependencies
-FROM golang:1.15-alpine AS builder
-
-RUN apk add --update ca-certificates git gcc g++ libc-dev
-WORKDIR /src/
-
-ENV GO111MODULE=on
-
-COPY go.mod .
-COPY go.sum .
-
+FROM alpine AS base
+RUN apk add --no-cache git terraform wget make openssh && \
+    wget -q https://github.com/goreleaser/goreleaser/releases/download/v1.19.2/goreleaser_1.19.2_x86_64.apk  && \
+    wget -q https://go.dev/dl/go1.19.2.linux-amd64.tar.gz && \
+    rm -rf /usr/local/go && \
+    tar -C /usr/local -xzf go1.19.2.linux-amd64.tar.gz && \
+	apk add --allow-untrusted goreleaser_1.19.2_x86_64.apk && \
+    mkdir -p /src/terraform-provider-artifactory
+ENV PATH=$PATH:/usr/local/go/bin
+WORKDIR /root
+
+FROM base as builder
+COPY . .
 RUN go mod download
-
-COPY pkg/ /src/pkg/
-COPY main.go /src/
-
-RUN CGO_ENABLED=0 GOOS=linux go build
-
-
-# Build the final image
-FROM hashicorp/terraform:0.13
-
-COPY --from=builder /src/terraform-provider-artifactory /root/.terraform.d/plugins/
+RUN make
+WORKDIR /root/v5-v6-migrator
+RUN make
+
+FROM hashicorp/terraform as plugin
+RUN adduser -S jfrog
+WORKDIR /home/jfrog
+COPY --from=builder /src/terraform-provider-artifactory/terraform-provider-artifactory /home/jfrog/.terraform.d/plugins/
+
+FROM alpine as migrator
+RUN adduser -S jfrog
+WORKDIR /home/jfrog
+COPY --from=builder /src/terraform-provider-artifactory/v5-v6-migrator/tf-v5-migrator /home/jfrog/tf-v5-migrator
+
+FROM alpine as importer
+RUN apk add --no-cache jq curl bash
+RUN adduser -S jfrog
+WORKDIR /home/jfrog
+USER jfrog
+COPY scripts/bulkimport.sh .
+ENTRYPOINT ./bulkimport.sh

sample.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     artifactory = {
       source  = "registry.terraform.io/jfrog/artifactory"
-      version = "7.10.1"
+      version = "8.9.2"
     }
   }
 }

scripts/bulkimport.sh

@@ -1,33 +1,240 @@
 #!/usr/bin/env bash
+${DEBUG:+set -x}
+set -e
+set -o errexit -o pipefail -o noclobber -o nounset
 
+HOME=${HOME:-"~"}
 
-function importRepos {
-  while read -r key type package; do
-	  cat <<-EOF
-		import {
-		  to = artifactory_${type,,}_${package,,}_repository.${key,,}
-		  id =  "${key,,}"
-		}
-		EOF
-	done < <(curl -snLf https://partnerenttest.jfrog.io/artifactory/api/repositories | jq -re '.[] | "\(.key) \(.type) \(.packageType)"'	)
-} && export -f importRepos
-
-function importUsers {
-  for i in {1..10}; do
-	  local username="username-${RANDOM}-${i}"
-	  cat <<-EOF
-		import {
-		  to = artifactory_user.${username}
-		  id = %s
-		}
-		EOF
-	done
-}
+command -v curl >/dev/null || (echo "You must install curl to proceed" >&2 && exit 1)
+command -v jq >/dev/null || (echo "You must install jq to proceed" >&2 && exit 1)
+command -v terraform >/dev/null || (echo "You must install terraform to proceed" >&2 && exit 1)
 
+function usage {
+	echo "${0} [{--users |-u} {--repos|-r} | {--groups|-g}] -h|--host https://\${host}" >&2
+	echo "duplicate resource declarations are de duped (see below)"
+	echo "example: ${0} -u --repos --users -h https://myartifactory.com > import.tf"
+	exit 1
+}
 
-resources=(importRepos importUsers importRepos)
-for f in $(echo "${resources[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' '); do
-  eval "${f}"
+resources=()
+while getopts urdga:-: OPT; do
+	# shellcheck disable=SC2154
+	if [ "$OPT" = "-" ]; then # long option: reformulate OPT and OPTARG
+		OPT="${OPTARG%%=*}"      # extract long option name
+		OPTARG="${OPTARG#$OPT}"  # extract long option argument (may be empty)
+		OPTARG="${OPTARG#=}"     # if long option argument, remove assigning `=`
+	fi
+	# shellcheck disable=SC2214
+	case "${OPT}" in
+	u | users)
+		resources+=(users)
+		;;
+	r | repos)
+		resources+=(repos)
+		;;
+	g | groups)
+		resources+=(groups)
+		;;
+	a | artifactory-url)
+		host="${OPTARG}"
+		grep -qE 'http(s)?://.*' <<<"${host}" || (echo "malformed url name: ${host}. must be of the form http(s)?://.*" && exit 1)
+		;;
+	??*)
+		usage
+		;;
+	*)
+		usage
+		;;
+	esac
 done
 
+function netrc_location {
+		case "$(uname)" in
+  		CYGWIN* | MSYS* | MINGW*)
+  			echo "$HOME/_netrc"
+  			;;
+  		Darwin* | Linux*)
+  			echo "$HOME/.netrc"
+  			;;
+  		*)
+  			echo "unsupported OS" >&2
+  			return 255
+  			;;
+  	esac
+}
+function assert_netrc {
+	local location
+	location="$(netrc_location)"
+	test -f "${location}" || touch "${location}"
+	echo "${location}"
+}
+
+function hasNetRcEntry {
+	local h="${1:?No host supplied}"
+
+	h="${h/https:\/\//}"
+  h="${h/http:\/\//}"
+  grep -qE "machine[ ]+${h}" "$(assert_netrc)"
+}
+
+function write_netrc {
+	local host="${1:?You must supply a host name}"
+	local netrc
+	netrc=$(assert_netrc)
+	read -r -p "please enter the username for ${host}: " username
+	read -rs -p "enter the api token (will not be echoed): " token
+	# append only to both files
+	cat <<-EOF  >> "${netrc}"
+		machine ${host}
+		login ${username}
+		password ${token}
+	EOF
+	echo "${host}"
+}
+
+# if they have no netrc file at all, create the file and add an entry
+if ! hasNetRcEntry "${host}" ; then
+	echo "added entry to $(write_netrc "${host}" )  needed for curl" >&2
+fi
+
+function repos {
+	# jq '.resources |map({type,name})' terraform.tfstate # make sure to not include anything already in state
+	# we'll make our internal jq structure match that of the tf state file so we can subtract them easy
+	local host="${1:?You must supply the artifactory host}"
+	# literally, usage of jq is 6x faster than bash/read/etc
+	# GET "${host}/artifactory/api/repositories" returns {key,type,packageType,url} where
+	# url) points to the UI for that resource??
+	# packageType) is cased ??
+	# type) is upcased and in "${host}/artifactory/api/repositories/${key}" it's not AND it's called rclass
+	local url="${host}/artifactory/api/repositories"
+	curl -snLf "${url}" | jq -re  --arg u "${url}" '.[] | "\($u)/\(.key)"' |
+		xargs -P 10 curl -snLf |
+			jq -sre '
+				group_by(.packageType == "docker" and .rclass == "local") |
+				(.[0] | map({
+						type: "artifactory_\(.rclass)_\(.packageType)_repository.\(.key)",
+						name: key
+					})
+				) +
+				(.[1] | map({
+						type: "artifactory_\(.rclass | ascii_downcase)_\(.packageType | ascii_downcase)_\(.dockerApiVersion | ascii_downcase)_repository.\(.key)",
+						name: key
+					})
+				) | .[] |
+"import {
+  to = \(.type)
+  id = \"\(.name)\"
+}"'
+} && export -f repos
+
+function accessTokens {
+	local host="${1:?You must supply the artifactory host}"
+	return 1
+	curl -snLf "${host}/artifactory/api/repositories/artifactory/api/security/token"
+}
+
+function ldapGroups {
+	local host="${1:?You must supply the artifactory host}"
+	return 1
+	curl -snLf "${host}/access/api/v1/ldap/groups"
+}
+
+function apiKeys {
+	local host="${1:?You must supply the artifactory host}"
+	return 1
+	curl -snLf "${host}/artifactory/api/security/apiKey"
+
+}
+
+function groups {
+	local host="${1:?You must supply the artifactory host}"
+	curl -snLf "${host}/artifactory/api/security/groups" |
+		jq -re '.[].name |
+  "import {
+    to = artifactory_group.\(.)
+    id = \"\(.)\"
+}"'
+}
+
+function certificates {
+	local host="${1:?You must supply the artifactory host}"
+	curl -snLf "${host}/artifactory/api/system/security/certificates/" |
+		jq -re '.[] |
+"import {
+	to = artifactory_certificate.\(.certificateAlias)
+	id = \"(.certificateAlias)\"
+}"'
+}
+
+function distributionPublicKeys {
+	local host="${1:?You must supply the artifactory host}"
+	# untested
+	return 1
+	curl -snLf "${host}/artifactory/api/security/keys/trusted" |
+	jq -re '.keys[] | "
+import {
+  to = artifactory_distribution_public_key.\(.alias)
+  id = \"\(.kid)\"
+}"'
+}
+
+function permissions {
+	#	these names have spaces in them
+	local host="${1:?You must supply the artifactory host}"
+	return 1 # untested
+	curl -snLf "${host}/artifactory/api/v2/security/permissions/" |
+		jq -re '.[] | select(.name | startswith("INTERNAL") | not) | "
+import {
+	to = artifactory_permission_target.\(.name)
+	id = \"\(.name)\"
+}"'
+}
+function keyPairs {
+	local host="${1:?You must supply the artifactory host}"
+	return 1 # untested
+	curl -snLf "${host}/artifactory/api/security/keypair/" |
+		jq -re '.[] | "
+import {
+  to = artifactory_keypair.\(.pairName)
+  id = \"\(.pairName)\"
+}"'
+}
+function users {
+	#	.name has values in it that artifactory will never accept, like email@. Not sure if in that case it should just be user-$RANDOM
+	local host="${1:?You must supply the artifactory host}"
+	curl -snLf "${host}/artifactory/api/security/users" | jq -re '.[] |
+	{user: .name | capture("(?<user>\\w+)@(?<domain>\\w+)").user, name}|
+	"import {
+  to = artifactory_user.\(.user)
+  id = \"\(.name)\"
+}"'
+}
+
+
+function output {
+	local host="${1:?You must supply artifactory host name}"
+# don't touch this heredoc if you want proper output format
+	cat <<-EOF
+		terraform {
+		  required_providers {
+		    artifactory = {
+		      source  = "registry.terraform.io/jfrog/artifactory"
+		      version = ">= 9.1.0"
+		    }
+		  }
+		}
+		provider "artifactory" {
+		  url = "${host}"
+		}
+
+		$(for f in "${@:2}"; do
+			eval "${f} ${host}"
+		done)
+	EOF
+}
 
+# shellcheck disable=SC2046
+output "${host}" $(echo "${resources[@]}" | tr ' ' '\n' | sort -u | tr '\n' ' ')
+# out="$RANDOM-out"
+#terraform plan -generate-config-out generated.tf -out "${out} -parallelism=10
+#terraform apply -parallelism=10

scripts/repodata.jq

@@ -0,0 +1,15 @@
+group_by(.packageType == "docker" and .rclass == "local") |
+(.[0] | map({
+		to: "artifactory_\(.rclass)_\(.packageType)_repository.\(.key)",
+		key
+	})
+) +
+(.[1] | map({
+		to: "artifactory_\(.rclass | ascii_downcase)_\(.packageType | ascii_downcase)_\(.dockerApiVersion | ascii_downcase)_repository.\(.key)",
+		key
+	})
+) | .[] |
+"import {
+  to = \(.to)
+  id = \"\(.key)\"
+}"