remove libdparse from token dump functionality

.github/workflows/default.yml

@@ -6,13 +6,14 @@ on:
   push:
     branches:
       - master
+      - replace_libdparse
 
 jobs:
   main:
     name: Run all tests
 
     # Only run for the main repository - not forks
-    if: ${{ github.repository == 'dlang-community/D-Scanner' }}
+    if: ${{ github.repository == 'Dlang-UPB/D-Scanner' }}
 
     # Run permutations of common os + host compilers
     strategy:
@@ -72,6 +73,10 @@ jobs:
            submodules: 'recursive'
            fetch-depth: 0
 
+      # Uncomment to get a ssh connection inside the GH Actions runner
+      #- name: Setup upterm session
+      #  uses: lhotari/action-upterm@v1
+
       # Install the host compiler (DMD or LDC)
       # Also grabs DMD for GDC to include dub + rdmd
       - name: Install ${{ matrix.compiler.version }}
@@ -128,6 +133,11 @@ jobs:
           fi
           "./bin/dscanner$EXE" --config .dscanner.ini --styleCheck src
 
+      - name: Run style checks
+        if: ${{ matrix.compiler.dmd == 'dmd' && matrix.build.type == 'make' }}
+        run: |
+            make style
+
       # Parse phobos to check for failures / crashes / ...
       - name: Checkout Phobos
         uses: actions/checkout@v2

.gitmodules

@@ -20,3 +20,6 @@
 [submodule "d-test-utils"]
 	path = d-test-utils
 	url = https://github.com/dlang-community/d-test-utils.git
+[submodule "dmd"]
+	path = dmd
+	url = [email protected]:dlang/dmd.git

README.md

@@ -1,13 +1,17 @@
-# D-Scanner
-
-[![CI status](https://travis-ci.org/dlang-community/D-Scanner.svg?branch=master)](https://travis-ci.org/dlang-community/D-Scanner/)
-[![Latest version](https://img.shields.io/dub/v/dscanner.svg)](http://code.dlang.org/packages/dscanner)
-[![License](https://img.shields.io/dub/l/dscanner.svg)](http://code.dlang.org/packages/dscanner)
+# D-Scanner fork that uses dmd as a library (WIP)
 
 D-Scanner is a tool for analyzing D source code
 
 ### Building and installing
-First make sure that you have all the source code. Run ```git submodule update --init --recursive```
+
+First, make sure that you have fetched the upstream: [email protected]:dlang-community/D-Scanner.git
+
+```
+git remote add upstream [email protected]:dlang-community/D-Scanner.git
+git fetch upstream
+```
+
+Secondly, make sure that you have all the source code. Run ```git submodule update --init --recursive```
 after cloning the project.
 
 To build D-Scanner, run ```make``` (or the build.bat file on Windows).

build.bat

@@ -18,8 +18,8 @@ if %githashsize% == 0 (
 	move /y bin\githash_.txt bin\githash.txt
 )
 
-set DFLAGS=-O -release -version=StdLoggerDisableWarning -Jbin %MFLAGS%
-set TESTFLAGS=-g -w -version=StdLoggerDisableWarning -Jbin
+set DFLAGS=-O -release -version=StdLoggerDisableWarning -version=CallbackAPI -version=DMDLIB -Jbin -Jdmd %MFLAGS%
+set TESTFLAGS=-g -w -version=StdLoggerDisableWarning -version=CallbackAPI -version=DMDLIB -Jbin -Jdmd
 set CORE=
 set LIBDPARSE=
 set STD=
@@ -31,6 +31,31 @@ set LIBDDOC=
 set STDXALLOCATOR=
 set STDXALLOCATORBLOCKS=
 
+set DMD_ROOT_SRC=
+for %%x in (dmd\src\dmd\common\*.d) do set DMD_ROOT_SRC=!DMD_ROOT_SRC! %%x
+for %%x in (dmd\src\dmd\root\*.d) do set DMD_ROOT_SRC=!DMD_ROOT_SRC! %%x
+
+set DMD_LEXER_SRC=^
+	dmd\src\dmd\console.d ^
+	dmd\src\dmd\entity.d ^
+	dmd\src\dmd\errors.d ^
+	dmd\src\dmd\file_manager.d ^
+	dmd\src\dmd\globals.d ^
+	dmd\src\dmd\id.d ^
+	dmd\src\dmd\identifier.d ^
+	dmd\src\dmd\lexer.d ^
+	dmd\src\dmd\tokens.d ^
+	dmd\src\dmd\utils.d
+
+set DMD_PARSER_SRC=^
+	dmd\src\dmd\astbase.d ^
+	dmd\src\dmd\parse.d ^
+	dmd\src\dmd\parsetimevisitor.d ^
+	dmd\src\dmd\transitivevisitor.d ^
+	dmd\src\dmd\permissivevisitor.d ^
+	dmd\src\dmd\strictvisitor.d ^
+	dmd\src\dmd\astenums.d
+
 for %%x in (src\dscanner\*.d) do set CORE=!CORE! %%x
 for %%x in (src\dscanner\analysis\*.d) do set ANALYSIS=!ANALYSIS! %%x
 for %%x in (libdparse\src\dparse\*.d) do set LIBDPARSE=!LIBDPARSE! %%x
@@ -49,14 +74,71 @@ for %%x in (stdx-allocator\source\stdx\allocator\building_blocks\*.d) do set STD
 if "%1" == "test" goto test_cmd
 
 @echo on
-%DC% %MFLAGS% %CORE% %STD% %LIBDPARSE% %LIBDDOC% %ANALYSIS% %INIFILED% %DSYMBOL% %CONTAINERS% %STDXALLOCATOR% %STDXALLOCATORBLOCKS% %DFLAGS% -I"libdparse\src" -I"dsymbol\src" -I"containers\src" -I"libddoc\src" -I"libddoc\common\source" -I"stdx-allocator\source" -ofbin\dscanner.exe
+%DC% %MFLAGS%^
+	%CORE%^
+	%STD%^
+	%LIBDPARSE%^
+	%LIBDDOC%^
+	%ANALYSIS%^
+	%INIFILED%^
+	%DSYMBOL%^
+	%CONTAINERS%^
+	%STDXALLOCATOR%^
+	%STDXALLOCATORBLOCKS%^
+	%DMD_ROOT_SRC%^
+	%DMD_LEXER_SRC%^
+	%DMD_PARSER_SRC%^
+	%DFLAGS%^
+	-I"libdparse\src"^
+	-I"dsymbol\src"^
+	-I"containers\src"^
+	-I"libddoc\src"^
+	-I"libddoc\common\source"^
+	-I"stdx-allocator\source"^
+	-I"dmd\src"^
+	-ofbin\dscanner.exe
+
 goto eof
 
 :test_cmd
 @echo on
 set TESTNAME="bin\dscanner-unittest"
-%DC% %MFLAGS% %STD% %LIBDPARSE% %LIBDDOC% %INIFILED% %DSYMBOL% %CONTAINERS% %STDXALLOCATOR% %STDXALLOCATORBLOCKS% -I"libdparse\src" -I"dsymbol\src" -I"containers\src" -I"libddoc\src" -I"stdx-allocator\source" -lib %TESTFLAGS% -of%TESTNAME%.lib
-if exist %TESTNAME%.lib %DC% %MFLAGS% %CORE% %ANALYSIS% %TESTNAME%.lib -I"src" -I"inifiled\source" -I"libdparse\src" -I"dsymbol\src" -I"containers\src" -I"libddoc\src" -I"libddoc\common\source" -I"stdx-allocator\source" -unittest %TESTFLAGS% -of%TESTNAME%.exe
+%DC% %MFLAGS% ^
+	%STD%^
+	%LIBDPARSE%^
+	%LIBDDOC%^
+	%INIFILED%^
+	%DSYMBOL%^
+	%CONTAINERS%^
+	%STDXALLOCATOR%^
+	%STDXALLOCATORBLOCKS%^
+	%DMD_ROOT_SRC%^
+	%DMD_LEXER_SRC%^
+	%DMD_PARSER_SRC%^
+	-I"libdparse\src"^
+	-I"dsymbol\src"^
+	-I"containers\src"^
+	-I"libddoc\src"^
+	-I"stdx-allocator\source"^
+	-I"dmd\src"^
+	-lib %TESTFLAGS%^
+	-of%TESTNAME%.lib
+if exist %TESTNAME%.lib %DC% %MFLAGS%^
+	%CORE%^
+	%ANALYSIS%^
+	%TESTNAME%.lib^
+	-I"src"^
+	-I"inifiled\source"^
+	-I"libdparse\src"^
+	-I"dsymbol\src"^
+	-I"containers\src"^
+	-I"libddoc\src"^
+	-I"libddoc\common\source"^
+	-I"stdx-allocator\source"^
+	-I"dmd\src"^
+	-unittest^
+	%TESTFLAGS%^
+	-of%TESTNAME%.exe
 if exist %TESTNAME%.exe %TESTNAME%.exe
 
 if exist %TESTNAME%.obj del %TESTNAME%.obj

dub.json

@@ -17,7 +17,10 @@
     "inifiled" : "~>1.3.1",
     "emsi_containers" : "~>0.8.0",
     "libddoc" : "~>0.8.0",
-    "stdx-allocator" : "~>2.77.5"
+    "stdx-allocator" : "~>2.77.5",
+    "dmd:root": "~master",
+    "dmd:lexer": "~master",
+    "dmd:parser": "~master"
   },
   "targetPath" : "bin",
   "stringImportPaths" : [

makefile

@@ -5,6 +5,31 @@ GIT ?= git
 DMD := $(DC)
 GDC := gdc
 LDC := ldc2
+DMD_ROOT_SRC := \
+	$(shell find dmd/src/dmd/common -name "*.d")\
+	$(shell find dmd/src/dmd/root -name "*.d")
+DMD_LEXER_SRC := \
+	dmd/src/dmd/console.d \
+	dmd/src/dmd/entity.d \
+	dmd/src/dmd/errors.d \
+	dmd/src/dmd/file_manager.d \
+	dmd/src/dmd/globals.d \
+	dmd/src/dmd/id.d \
+	dmd/src/dmd/identifier.d \
+	dmd/src/dmd/lexer.d \
+	dmd/src/dmd/tokens.d \
+	dmd/src/dmd/utils.d \
+	$(DMD_ROOT_SRC)
+
+DMD_PARSER_SRC := \
+	dmd/src/dmd/astbase.d \
+	dmd/src/dmd/parse.d \
+	dmd/src/dmd/parsetimevisitor.d \
+	dmd/src/dmd/transitivevisitor.d \
+	dmd/src/dmd/permissivevisitor.d \
+	dmd/src/dmd/strictvisitor.d \
+	dmd/src/dmd/astenums.d \
+	$(DMD_LEXER_SRC)
 
 LIB_SRC := \
 	$(shell find containers/src -name "*.d")\
@@ -14,7 +39,8 @@ LIB_SRC := \
 	$(shell find libdparse/src/dparse/ -name "*.d")\
 	$(shell find libddoc/src -name "*.d") \
 	$(shell find libddoc/common/source -name "*.d") \
-	$(shell find stdx-allocator/source -name "*.d")
+	$(shell find stdx-allocator/source -name "*.d") \
+	$(DMD_PARSER_SRC)
 PROJECT_SRC := $(shell find src/ -name "*.d")
 SRC := $(LIB_SRC) $(PROJECT_SRC)
 
@@ -44,24 +70,25 @@ INCLUDE_PATHS = \
 	-Icontainers/src \
 	-Ilibddoc/src \
 	-Ilibddoc/common/source \
-	-Istdx-allocator/source
+	-Istdx-allocator/source \
+	-Idmd/src
 
-DMD_VERSIONS = -version=StdLoggerDisableWarning
+DMD_VERSIONS = -version=StdLoggerDisableWarning -version=CallbackAPI -version=DMDLIB
 DMD_DEBUG_VERSIONS = -version=dparse_verbose
-LDC_VERSIONS = -d-version=StdLoggerDisableWarning
+LDC_VERSIONS = -d-version=StdLoggerDisableWarning -d-version=CallbackAPI -d-version=DMDLIB
 LDC_DEBUG_VERSIONS = -d-version=dparse_verbose
-GDC_VERSIONS = -fversion=StdLoggerDisableWarning
+GDC_VERSIONS = -fversion=StdLoggerDisableWarning -fversion=CallbackAPI -fversion=DMDLIB
 GDC_DEBUG_VERSIONS = -fversion=dparse_verbose
 
-DC_FLAGS += -Jbin
+DC_FLAGS += -Jbin -Jdmd
 override DMD_FLAGS += $(DFLAGS) -w -release -O -od${OBJ_DIR}
 override LDC_FLAGS += $(DFLAGS) -O5 -release -oq
 override GDC_FLAGS += $(DFLAGS) -O3 -frelease
 
-DC_TEST_FLAGS += -g -Jbin
+DC_TEST_FLAGS += -g -Jbin -Jdmd
 override DMD_TEST_FLAGS += -w
 
-DC_DEBUG_FLAGS := -g -Jbin
+DC_DEBUG_FLAGS := -g -Jbin -Jdmd
 
 ifeq ($(DC), $(filter $(DC), dmd ldmd2 gdmd))
 	VERSIONS := $(DMD_VERSIONS)
@@ -82,12 +109,10 @@ else ifneq (,$(findstring gdc, $(DC)))
 	DC_TEST_FLAGS += $(GDC_TEST_FLAGS) -funittest
 	WRITE_TO_TARGET_NAME = -o $@
 endif
-
 SHELL:=/usr/bin/env bash
 
 GITHASH = bin/githash.txt
 
-
 $(OBJ_DIR)/$(DC)/%.o: %.d
 	${DC} ${DC_FLAGS} ${VERSIONS} ${INCLUDE_PATHS} -c $< ${WRITE_TO_TARGET_NAME}
 
@@ -150,3 +175,42 @@ report: all
 
 release:
 	./release.sh
+
+# Add source files here as we transition to DMD-as-a-library
+STYLE_CHECKED_SRC := \
+	src/dscanner/imports.d \
+	src/dscanner/main.d
+
+style:
+	@echo "Check for trailing whitespace"
+	grep -nr '[[:blank:]]$$' ${STYLE_CHECKED_SRC}; test $$? -eq 1
+
+	@echo "Enforce whitespace before opening parenthesis"
+	grep -nrE "\<(for|foreach|foreach_reverse|if|while|switch|catch|version)\(" ${STYLE_CHECKED_SRC} ; test $$? -eq 1
+
+	@echo "Enforce no whitespace after opening parenthesis"
+	grep -nrE "\<(version) \( " ${STYLE_CHECKED_SRC} ; test $$? -eq 1
+
+	@echo "Enforce whitespace between colon(:) for import statements (doesn't catch everything)"
+	grep -nr 'import [^/,=]*:.*;' ${STYLE_CHECKED_SRC} | grep -vE "import ([^ ]+) :\s"; test $$? -eq 1
+
+	@echo "Check for package wide std.algorithm imports"
+	grep -nr 'import std.algorithm : ' ${STYLE_CHECKED_SRC} ; test $$? -eq 1
+
+	@echo "Enforce Allman style"
+	grep -nrE '(if|for|foreach|foreach_reverse|while|unittest|switch|else|version) .*{$$' ${STYLE_CHECKED_SRC}; test $$? -eq 1
+
+	@echo "Enforce do { to be in Allman style"
+	grep -nr 'do *{$$' ${STYLE_CHECKED_SRC} ; test $$? -eq 1
+
+	@echo "Enforce no space between assert and the opening brace, i.e. assert("
+	grep -nrE 'assert +\(' ${STYLE_CHECKED_SRC} ; test $$? -eq 1
+
+	@echo "Enforce space after cast(...)"
+	grep -nrE '[^"]cast\([^)]*?\)[[:alnum:]]' ${STYLE_CHECKED_SRC} ; test $$? -eq 1
+
+	@echo "Enforce space between a .. b"
+	grep -nrE '[[:alnum:]][.][.][[:alnum:]]|[[:alnum:]] [.][.][[:alnum:]]|[[:alnum:]][.][.] [[:alnum:]]' ${STYLE_CHECKED_SRC}; test $$? -eq 1
+
+	@echo "Enforce space between binary operators"
+	grep -nrE "[[:alnum:]](==|!=|<=|<<|>>|>>>|^^)[[:alnum:]]|[[:alnum:]] (==|!=|<=|<<|>>|>>>|^^)[[:alnum:]]|[[:alnum:]](==|!=|<=|<<|>>|>>>|^^) [[:alnum:]]" ${STYLE_CHECKED_SRC}; test $$? -eq 1