Merge pull request #746 from myk002/myk_symbols_gen

automated symbol generation scripts for Linux and Windows

symbols.xml

@@ -119,8 +119,13 @@
 
     <!-- Windows logo by Microsoft -->
 
+    <symbol-table name='v0.00.00 win64 LOCAL' os-type='windows'>
+        <binary-timestamp value='0x00000000'/>
+    </symbol-table>
+
     <symbol-table name='v0.50.12 win64 STEAM' os-type='windows'>
         <binary-timestamp value='0x65E8286D'/>
+
         <global-address name='texture' value='0x1421d4670'/>
         <global-address name='ui_look_list' value='0x1421d4658'/>
         <global-address name='created_item_subtype' value='0x1421d4628'/>
@@ -296,6 +301,7 @@
         <global-address name='cursor' value='0x1413f81c8'/>
         <global-address name='selection_rect' value='0x1413f8188'/>
         <global-address name='start_dwarf_count' value='0x1413f8174'/>
+
         <vtable-address name='MacroScreenLoad' value='0x14129db88'/>
         <vtable-address name='MacroScreenSave' value='0x14129dbe0'/>
         <vtable-address name='abstract_building_counting_housest' value='0x1411c0290'/>
@@ -3205,6 +3211,10 @@
 
     <!-- Tux logo by the Linux guys :) -->
 
+    <symbol-table name='v0.00.00 linux64 LOCAL' os-type='linux'>
+        <md5-hash value='00000000000000000000000000000000'/>
+    </symbol-table>
+
     <symbol-table name='v0.50.12 linux64 STEAM' os-type='linux'>
         <md5-hash value='c7dcc28bc714daff32f6f53c95542faf'/>
 

symbols_gen_common.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# This script is intended to be sourced from the other symbols_gen scripts.
+# SCRIPT_DIR must be defined before sourcing this script.
+
+DF_VER="$1"
+DF_TYPE="$2"
+DF_DIR="$3"
+
+SYMBOLS_XML="${SCRIPT_DIR}/symbols.xml"
+
+if [ -z "${DF_VER}" ]; then
+    echo "DF version (arg 1) must be specified"
+    exit 1
+elif [ -z "${DF_TYPE}" ]; then
+    echo "DF distribution type (arg 2) must be specified"
+    exit 1
+elif [ ! -w "${SYMBOLS_XML}" ]; then
+    echo "DFHack symbols not found or not writable: ${SYMBOLS_XML}"
+    exit 1
+fi
+
+write_symbol_table() {
+    platform="$1"
+    os_type="$2"
+    id_elem="$3"
+    offsets="$4"
+    vtables="$5"
+
+    tmpfile=`mktemp`
+
+    formatted_offsets=`echo "${offsets}" | sed "s/^/        /"`
+    formatted_vtables=`echo "${vtables}" | sed "s/^/        /"`
+
+    cat >${tmpfile} <<EOF
+    <symbol-table name='v0.${DF_VER} ${platform} ${DF_TYPE}' os-type='${os_type}'>
+        ${id_elem}
+
+${formatted_offsets}
+
+${formatted_vtables}
+    </symbol-table>
+EOF
+
+    start_pattern="<symbol-table.*${platform} ${DF_TYPE}"
+    end_pattern="<\/symbol-table>"
+    sed -i "/${start_pattern}/,/${end_pattern}/{/${end_pattern}/{x;r ${tmpfile}
+    D};d}" "${SYMBOLS_XML}"
+    rm "${tmpfile}"
+}

symbols_gen_linux.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# This script overwrites the linux symbol-table element in library/xml/symbols.xml
+# that has the specified $DF_TYPE with the current symbols for the DF version
+# $DF_VER installed in the given directory. $DF_TYPE can be STEAM, ITCH, CLASSIC,
+# or LOCAL.
+#
+# Pre-requisites:
+#
+#   - DFHack is already installed in the given DF directory
+#   - exactly one symbol-table element exists for linux64 $DF_TYPE in symbols.xml
+#   - ansifilter is installed on the host system
+#
+# Syntax:
+#
+#   path/to/library/xml/symbols_gen_linux.sh <DF version> <DF type> [<DF dir>]
+#
+# If the DF dir is not specified, it defaults to the location that cmake is
+# configured to install DFHack to in the "../../build" dir cmake configuration.
+# This default only works if this script is run from a configured DFHack source
+# tree.
+#
+# The script will act upon the symbols.xml in the same directory as this script,
+# regardless of what the the CWD is when this script is run.
+
+SCRIPT_DIR=`dirname $0`
+
+. ${SCRIPT_DIR}/symbols_gen_common.sh
+
+if [ -z "${DF_DIR}" ]; then
+    build_dir=`cd "${SCRIPT_DIR}/../../build" && pwd`
+    if [ -n ${build_dir} ]; then
+        install_prefix=`grep CMAKE_INSTALL_PREFIX:PATH= ${build_dir}/CMakeCache.txt | cut -d= -f2`
+        DF_DIR=`cd ${build_dir} && cd ${install_prefix} && pwd`
+    fi
+fi
+
+DWARFORT="${DF_DIR}/dwarfort"
+ANSIFILTER="ansifilter"
+DFHACK_RUN="${DF_DIR}/dfhack-run"
+
+if [ ! -x "${DWARFORT}" ]; then
+    echo "DF not found or not executable: ${DWARFORT}"
+    exit 1
+elif [ ! -x "${DFHACK_RUN}" ]; then
+    echo "DFHack must be installed in ${DF_DIR}"
+    exit 1
+elif [ -z "$(which ${ANSIFILTER})" ]; then
+    echo "required dependency not installed: ${ANSIFILTER}"
+    exit 1
+fi
+
+df_hash=`md5sum "${DWARFORT}" | cut -d" " -f1`
+hash_elem="<md5-hash value='${df_hash}'/>"
+
+write_symbol_table linux64 linux "${hash_elem}" "" ""
+cp "${SYMBOLS_XML}" "${DF_DIR}/hack/symbols.xml"
+(cd "${DF_DIR}" && DFHACK_DISABLE_CONSOLE=1 ./dfhack >/dev/null 2>&1) &
+
+offsets=''
+while [ -z "${offsets}" ]; do
+    sleep 0.5
+    echo "waiting for DF to start"
+    offsets=`"${DFHACK_RUN}" devel/dump-offsets | ${ANSIFILTER} | fgrep global-address`
+done
+vtables=`"${DFHACK_RUN}" devel/scan-vtables | ${ANSIFILTER} | LANG=C sort | fgrep vtable-address`
+write_symbol_table linux64 linux "${hash_elem}" "$offsets" "$vtables"
+
+"${DFHACK_RUN}" die
+echo "done"

symbols_gen_windows.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# This script overwrites the windows symbol-table element in library/xml/symbols.xml
+# that has the specified $DF_TYPE with the current symbols for the DF version $DF_VER
+# executable installed in the given DF dir. $DF_TYPE can be STEAM, ITCH, CLASSIC, or
+# LOCAL.
+#
+# Pre-requisites:
+#
+#   - exactly one symbol-table element exists for win64 $DF_TYPE in symbols.xml
+#   - python and ruby are installed
+#   - the df_misc repo is checked out in the df_misc subdirectory of the current dir
+#     - git clone --depth=1 https://github.com/DFHack/df_misc.git
+#   - the metasm repo is checked out in the metasm subdirectory of the current dir
+#     - git clone --depth=1 https://github.com/jjyg/metasm.git
+#   - pefile is installed or pefile.py is available in the pefile subdirectory of the current dir
+#     - wget 'https://github.com/erocarrera/pefile/releases/download/v2023.2.7/pefile-2023.2.7.tar.gz
+#     - mkdir pefile && tar xzf pefile-2023.2.7.tar.gz -C pefile --strip-components=1
+#
+# Syntax:
+#
+#   path/to/library/xml/symbols_gen_windows.sh <DF version> <DF type> [<DF dir>]
+#
+# If not specified, the DF dir defaults to:
+#   $HOME/.steam/bin32/steamapps/content/app_975370/depot_975372
+# which is where it shows up when the Windows binary manifest is downloaded with the
+# Steam client. The only file that must be in this directory is Dwarf Fortress.exe.
+#
+# The script will act upon the symbols.xml in the same directory as this script,
+# regardless of what the the CWD is when this script is run.
+
+SCRIPT_DIR=`dirname $0`
+
+. ${SCRIPT_DIR}/symbols_gen_common.sh
+
+DF_DIR=${DF_DIR:-$HOME/.steam/bin32/steamapps/content/app_975370/depot_975372}
+DF_EXE="${DF_DIR}/Dwarf Fortress.exe"
+
+DF_MISC_DIR="$(pwd)/df_misc"
+METASM_DIR="$(pwd)/metasm"
+
+if which pefile >/dev/null 2>&1; then
+    PEFILE=pefile
+elif [ -r pefile/pefile.py ]; then
+    PEFILE="cd pefile && python pefile.py"
+fi
+
+if [ ! -x "${DF_EXE}" ]; then
+    echo "DF executable not found: ${DF_EXE}"
+    exit 1
+elif [ ! -d "${DF_MISC_DIR}" ]; then
+    echo "required dependency not in subdir: df_misc"
+    exit 1
+elif [ ! -d "${METASM_DIR}" ]; then
+    echo "required dependency not in subdir: metasm"
+    exit 1
+fi
+
+pe_timestamp=`eval ${PEFILE} \"${DF_EXE}\" | fgrep TimeDateStamp | head -n1 | sed 's/  */ /g' | cut -d" " -f4`
+
+timestamp_elem="<binary-timestamp value='${pe_timestamp}'/>"
+
+offsets=`cd df_misc && RUBYLIB="${METASM_DIR}" ruby dump_df_globals.rb "${DF_EXE}"`
+vtables=`cd df_misc && RUBYLIB="${METASM_DIR}" ruby scan_vtable.rb "${DF_EXE}"`
+
+write_symbol_table win64 windows "${timestamp_elem}" "$offsets" "$vtables"
+echo "done"