From d581d8bc809c53d466cff5308e9e8327a79f5f76 Mon Sep 17 00:00:00 2001 From: Roman Leshchinskiy Date: Sun, 11 Sep 2022 21:26:00 +0000 Subject: [PATCH] Add index-llvm.sh script This runs the indexer on the LLVM source distribution. I think it's a good demo. `index-llvm.sh --help` gives a complete example. I'm not sure where the script should live, though. --- index-llvm.sh | 212 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100755 index-llvm.sh diff --git a/index-llvm.sh b/index-llvm.sh new file mode 100755 index 000000000..ed01064bd --- /dev/null +++ b/index-llvm.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +set -e +shopt -s extglob +shopt -s inherit_errexit + +fatal() { + echo "$@" 1>&2 + exit 1 +} + +usage() { + cat << EOF +Usage: $(basename "$0") FLAGS SOURCE_DIR TARGET_DIR + + Index LLVM source distribution located in SOURCE_DIR into TARGET_DIR + +Example: + mkdir -p /tmp/llvm \\ + && ( wget -O - https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.6/llvm-14.0.6.src.tar.xz \\ + | tar -C /tmp/llvm -xJ ) \\ + && $0 -j$(grep -c '^processor' /proc/cpuinfo) /tmp/llvm/llvm-14.0.6.src /tmp/llvm + + +Available options: + --glean DIR Specify Glean source directory + -jN | --jobs N Use N concurrent indeces + --db-root DIR Store the generated database in DIR + (default TARGET_DIR/db) + --db DB Use provided name for the database + (default llvm/VERSION) + --overwrite Overwrite the database if it already exists + --schema DIR Use the schema in DIR (default is the schema in + Glean's source tree) +EOF +} + +argerror() { + echo "$@" >&2 + usage >&2 + exit 1 +} + +if [[ "$1" == "--help" ]] +then + usage + exit 0 +fi + +MAKE_ARGS=() +EXTRA_GLEAN_ARGS=() +EXTRA_GLEAN_INDEX_ARGS=() +VERBOSITY=0 +GLEAN_DB_ROOT= +GLEAN_SCHEMA= +GLEAN_DIR= + +while true +do + case "$1" in + (-j+([0-9])) + MAKE_ARGS+=("$1") + EXTRA_GLEAN_INDEX_ARGS+=("$1") + shift + ;; + -j|--jobs) + MAKE_ARGS+=("$1") + EXTRA_GLEAN_INDEX_ARGS+=("$1") + case "$2" in + (+([0-9])) + MAKE_ARGS+=("$2") + EXTRA_GLEAN_INDEX_ARGS+=("$2") + shift 2 + ;; + *) + argerror "Invalid number of jobs" + ;; + esac + ;; + -v|--verbose) + VERBOSITY=1 + EXTRA_GLEAN_INDEX_ARGS+=("-v") + shift + ;; + --db-root) + GLEAN_DB_ROOT="$2" + shift 2 + ;; + --schema) + GLEAN_SCHEMA="$2" + shift 2 + ;; + --db) + GLEAN_DB="$2" + shift 2 + ;; + --overwrite) + OVERWRITE=yes + shift + ;; + --glean) + GLEAN_DIR="$2" + shift 2 + ;; + -*) + argerror "Unsupported option $1" + ;; + *) + break + ;; + esac +done + +if [[ -z "$1" ]] +then + argerror "No source directory specified" +fi + +SOURCE_DIR="$1" +shift + +if [[ -z "$1" ]] +then + argerror "No destination directory specified" +fi + +OUTPUT_DIR="$1" +shift + +if [[ $# -ne 0 ]] +then + argerror "Extra arguments" +fi + +if [[ -z "${GLEAN_DIR}" ]] +then + # Look for any ancestor directory with glean.cabal.in - this allows us to + # move the script within the source tree + dir=$(dirname "$0") + while [[ "${dir}" != "/" ]] ; do + if [[ -f "${dir}"/glean.cabal.in ]] + then + GLEAN_DIR="${dir}" + break + fi + dir=$(dirname "${dir}") + done + if [[ -z "${GLEAN_DIR}" ]] ; then + fatal "Couldn't local Glean source directory, please specify --glean" + fi +fi + +BUILD_DIR="${OUTPUT_DIR}"/build +mkdir -p "${BUILD_DIR}" +GLEAN_DB_ROOT="${GLEAN_DB_ROOT:-${OUTPUT_DIR}/db}" +mkdir -p "${GLEAN_DB_ROOT}" + +GLEAN_SCHEMA="${GLEAN_SCHEMA:-${GLEAN_DIR}/glean/schema/source}" + +# FD 5 is where we redirect all output +if [[ "${VERBOSITY}" == "1" ]] ; then + exec 5>&0 +else + exec 5> "${OUTPUT_DIR}/index-llvm.log" +fi + +echo "Building glean-clang" +make MODE=opt glean-clang >&5 + +requirebin() { + tmp="$("${GLEAN_DIR}/quick.sh" MODE=opt list-bin "$1")" + if ! [[ -x "${tmp}" ]] ; then + fatal "$1 doesn't exist at ${tmp}" + fi + echo "${tmp}" +} + +CLANG_INDEX=$(requirebin glean-clang:clang-index) +CLANG_DERIVE=$(requirebin glean-clang:clang-derive) +GLEAN=$(requirebin glean:glean) + +echo "Setting up ${BUILD_DIR}" +mkdir -p "${BUILD_DIR}" >&5 +"${CMAKE:-cmake}" \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=1 \ + -DLLVM_INCLUDE_BENCHMARKS=OFF -DLLVM_TARGETS_TO_BUILD=X86 \ + -S "${SOURCE_DIR}" -B "${BUILD_DIR}" >&5 + +echo "Generating LLVM code" +(cd "${BUILD_DIR}" && \ + make "${MAKE_ARGS[@]}" intrinsics_gen X86CommonTableGen WindresOptsTableGen \ + OtoolOptsTableGen DllOptionsTableGen acc_gen omp_gen) >&5 + +if [[ -z "${GLEAN_DB}" ]] ; then + GLEAN_DB=llvm/$(sed -n '/^Version/ {s/^Version: //p;q}' "${BUILD_DIR}/llvm.spec") +fi + +if [[ -n "${OVERWRITE}" ]] ; then + rm -rf "${GLEAN_DB_ROOT:?}/${GLEAN_DB}" +fi + +echo "Indexing ${GLEAN_DB} in ${GLEAN_DB_ROOT}" +"${GLEAN}" "${EXTRA_GLEAN_ARGS[@]}" \ + --schema "${GLEAN_SCHEMA}" --db-root "${GLEAN_DB_ROOT}" \ + index --db "${GLEAN_DB}" cpp-cmake \ + --indexer "${CLANG_INDEX}" --deriver "${CLANG_DERIVE}" --cdb "${BUILD_DIR}" \ + --verbose --progress "${EXTRA_GLEAN_INDEX_ARGS[@]}" "${SOURCE_DIR}" 2>&5