Skip to content

Commit 091c32a

Browse files
seanshparkseanshpark
committed
DRAFT initial circle-mlir project
on-going draft to introduce initial circle-mlir project. Signed-off-by: SaeHie Park <[email protected]>
1 parent edb2598 commit 091c32a

File tree

61 files changed

+3054
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+3054
-0
lines changed

‎circle-mlir/Makefile.aa

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
all: cfg debug test install
2+
3+
cfg:
4+
Python3_ROOT_DIR=/usr/bin cmake -B build/debug -S ./ \
5+
-DCMAKE_INSTALL_PREFIX=build/debug.install \
6+
-DCMAKE_BUILD_TYPE=Debug \
7+
-DCIRCLE_MLIR_WORKDIR=/workdir
8+
9+
cc:
10+
Python3_ROOT_DIR=/usr/bin cmake -B build/debug -S ./ \
11+
-DCMAKE_INSTALL_PREFIX=build/debug.install \
12+
-DCMAKE_BUILD_TYPE=Debug
13+
14+
debug:
15+
cmake --build build/debug -j4
16+
17+
test:
18+
CTEST_OUTPUT_ON_FAILURE=1 cmake --build build/debug --verbose -- test
19+
20+
install:
21+
cmake --build build/debug -j4 -- install
22+

‎circle-mlir/Makefile.sample

+72
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,12 @@ endif
1515
# TODO error handle if not found
1616
PYTHON3_PATH=$(shell dirname $(PYTHON3_CMD))
1717

18+
# NOTE CIRCLEM_LIR_XXX is used for CMakeLists
19+
# CIRCLEMLIR_XXX is used in this Makefile
20+
1821
CIRCLEMLIR_BUILD_DEBUG?=build/debug
1922
CIRCLEMLIR_BUILD_REL?=build/release
23+
CIRCLEMLIR_BUILD_COV?=build/coverage
2024
CIRCLEMLIR_EXTS_DEBUG?=build/externals/debug
2125
CIRCLEMLIR_EXTS_REL?=build/externals/release
2226

@@ -40,7 +44,16 @@ help:
4044
@echo "make prepr : prepare externals for release (needed only once)"
4145
@echo "make cfgr : configure circle-mlir for release build"
4246
@echo "make rel : build for release"
47+
@echo "make prepcov : prepare submodules for coverage test (needed only once)"
48+
@echo "make cfgcov : configure circle-mlir for debug build with coverage test"
49+
@echo "make debugcov : build for test coverage"
50+
@echo "make testcov : run coverage test"
51+
@echo "make gencov : generate test coverage report"
52+
@echo "make cleancov : clean test coverage build"
4353
@echo "make testr : test for release"
54+
@echo "make cfgdi : configure circle-mlir for debug build in Docker image"
55+
@echo "make cfgcovdi : configure circle-mlir for debug build with coverage test in Docker image"
56+
@echo "make cfgri : configure circle-mlir for release build in Docker image"
4457
@echo "make cleanr : clean release build"
4558
@echo "make cleanall : clean all build including overlay, externals"
4659

@@ -86,6 +99,38 @@ clean:
8699
rm -f $(CIRCLEMLIR_BUILD_DEBUG)/CMakeCache.txt
87100
rm -rf $(CIRCLEMLIR_BUILD_DEBUG)/circle-mlir/
88101

102+
#-------------------------------------------------------------------------------
103+
# for debug test coverage
104+
105+
prepcov: _mkbuildcov
106+
Python3_ROOT_DIR=$(CIRCLEMLIR_PY3_ROOT) \
107+
cmake -B $(CIRCLEMLIR_EXTS_DEBUG) -S ./externals -DCMAKE_BUILD_TYPE=Release
108+
cmake --build $(CIRCLEMLIR_EXTS_DEBUG) -j$(CIRCLEMLIR_BUILD_JOBS)
109+
110+
cfgcov: _mkbuildcov
111+
cmake -B $(CIRCLEMLIR_BUILD_COV) -S ./ \
112+
-DCIRCLE_MLIR_EXTERNALS=$(CIRCLEMLIR_EXTS_DEBUG) \
113+
-DONNX2CIRCLE_TEST_MODELS_SINGLE=ON \
114+
-DENABLE_COVERAGE=ON
115+
116+
debugcov:
117+
CM_PASS_DUMP=2 \
118+
cmake --build $(CIRCLEMLIR_BUILD_COV) -j$(CIRCLEMLIR_BUILD_JOBS)
119+
120+
# NOTE to configure in Docker, use "make cfgcovdi"
121+
122+
testcov:
123+
CM_PASS_DUMP=2 \
124+
CTEST_OUTPUT_ON_FAILURE=1 \
125+
cmake --build $(CIRCLEMLIR_BUILD_COV) --verbose -- test
126+
127+
gencov:
128+
bash infra/tools/gen-coverage-report circle-mlir
129+
130+
cleancov:
131+
rm -f $(CIRCLEMLIR_BUILD_COV)/CMakeCache.txt
132+
rm -rf $(CIRCLEMLIR_BUILD_COV)/circle-mlir/
133+
89134
#-------------------------------------------------------------------------------
90135
# for release
91136

@@ -109,6 +154,33 @@ cleanr:
109154
rm -f $(CIRCLEMLIR_BUILD_REL)/CMakeCache.txt
110155
rm -rf $(CIRCLEMLIR_BUILD_REL)/circle-mlir/
111156

157+
#-------------------------------------------------------------------------------
158+
# for debug build in Docker
159+
#
160+
# no need to make for overlay, prep as prepared in Docker image
161+
# run make for 'cfgdi'
162+
# then make for 'debug', 'test'
163+
164+
cfgdi: _mkbuild
165+
cmake -B $(CIRCLEMLIR_BUILD_DEBUG) -S ./ \
166+
-DONNX2CIRCLE_TEST_MODELS_SINGLE=ON \
167+
-DCMAKE_BUILD_TYPE=Debug \
168+
-DCIRCLE_MLIR_WORKDIR=/workdir
169+
170+
# for test converage build in Docker
171+
cfgcovdi: _mkbuildcov
172+
cmake -B $(CIRCLEMLIR_BUILD_COV) -S ./ \
173+
-DONNX2CIRCLE_TEST_MODELS_SINGLE=ON \
174+
-DCIRCLE_MLIR_WORKDIR=/workdir \
175+
-DENABLE_COVERAGE=ON
176+
177+
# for release build in Docker
178+
179+
cfgri: _mkbuild
180+
cmake -B $(CIRCLEMLIR_BUILD_REL) -S ./ \
181+
-DONNX2CIRCLE_TEST_MODELS_SINGLE=ON \
182+
-DCMAKE_BUILD_TYPE=Release \
183+
-DCIRCLE_MLIR_WORKDIR=/workdir
112184

113185
#-------------------------------------------------------------------------------
114186

‎circle-mlir/README.md

+172
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,175 @@
11
# circle-mlir
22

33
Circle MLIR dialect and tools
4+
5+
## Tools provided
6+
7+
_onnx2circle_
8+
- conversion tool of ONNX to Circle model for `compiler`
9+
- to replace not-maintained-anymore onnx-tensorflow package
10+
11+
## How to build
12+
13+
Use provided `Makefile.sample` or create your own `Makefile`
14+
```
15+
ln -s Makefile.sample Makefile
16+
```
17+
- `Makefile` is in `.gitignore` to let developers use own Makefile.
18+
19+
### Prerequisite
20+
21+
```
22+
sudo apt-get install build-essential cmake git fakeroot
23+
sudo apt-get install autoconf automake libtool unzip wget
24+
sudo apt-get install devscripts debmake debhelper lcov
25+
sudo apt-get install python3 python3-pip python3-venv python3-dev python3-all dh-python
26+
27+
python3 -m pip install --upgrade pip setuptools
28+
python3 -m pip install yapf==0.43.0 numpy==1.26.4 h5py==3.8.0 einops
29+
```
30+
31+
### Prepare externals
32+
33+
### Debug build
34+
35+
Prepare overlay
36+
```
37+
make overlay
38+
```
39+
40+
Build submodules in venv
41+
```
42+
source infra/overlay/venv/bin/activate
43+
make prep
44+
```
45+
NOTE `llvm-project` is built as `Debug` which may require 32G or more RAM.
46+
- if build fails for some reason, please change back to
47+
`-DCMAKE_BUILD_TYPE=Release` in `prep:` target in `Makefile.sample` file.
48+
- build and test needs venv python packages.
49+
50+
NOTE `overlay` and `submodules` builds are needed only once.
51+
52+
Configure and build
53+
```
54+
make cfg
55+
make debug
56+
```
57+
58+
Test build
59+
```
60+
make test
61+
```
62+
- optionally, set `ONE_COMPILER_ROOT` to alternate PATH for local ONE build
63+
```
64+
ONE_COMPILER_ROOT=/home/user/one/build/install make test
65+
```
66+
67+
To clean up existing build results
68+
```
69+
make clean
70+
```
71+
72+
To clean up also `overlay` and `submodules`
73+
```
74+
make cleanall
75+
```
76+
- NOTE when using `CIRCLE_MLIR_LOCALINST`, need to manually clean up this folder
77+
78+
### Release build
79+
80+
Release build is available as follows.
81+
Others not mentioned are same as above Debug build.
82+
83+
Build submodules in venv
84+
```
85+
source infra/overlay/venv/bin/activate
86+
make prepr
87+
deactivate
88+
```
89+
90+
Configure and build
91+
```
92+
make cfgr
93+
make rel
94+
```
95+
96+
Test build
97+
```
98+
make testr
99+
```
100+
101+
### Test coverage
102+
103+
To get test coverage report, run as following commands.
104+
- assume you already have done `make overlay` and `make prepcov`
105+
- you can skip `make prepcov` step if you are using local installation with `CIRCLE_MLIR_LOCALINST`
106+
- or you can reuse `CIRCLE_MLIR_LOCALINST` for existing debug or release build submodules with
107+
`cfgcov` target such as `CIRCLE_MLIR_LOCALINST=$(pwd)/build/debug/submodules make cfgcov`
108+
```
109+
source infra/overlay/venv/bin/activate
110+
make cfgcov
111+
deactivate
112+
113+
make debugcov
114+
make testcov
115+
make gencov
116+
```
117+
118+
Open `converage/html/index.html` file in web browser to see the reports.
119+
120+
To generate from second run and so on in your local machine, you will have to
121+
remove existing files before running `gencov`
122+
```
123+
rm -rf coverage
124+
make gencov
125+
```
126+
127+
To run this with Docker image, use `cfgcovdi` target instead of `cfgcov`.
128+
```
129+
make cfgcovdi
130+
make debugcov
131+
make testcov
132+
make gencov
133+
```
134+
135+
136+
## Local format check
137+
138+
Install prerequiste package.
139+
```
140+
sudo apt-get install clang-format-12 python3 python3-pip
141+
python3 -m pip install yapf==0.32.0
142+
```
143+
144+
Run format checker.
145+
```
146+
bash ./infra/tools/format
147+
```
148+
or with `Makefile` from `Makefile.sample`
149+
```
150+
make format
151+
```
152+
153+
## Dump debug logs
154+
155+
To see logs during conversion with `onnx2circle` tool, set `CM_PASS_DUMP=1` for
156+
preprocessing ONNX and ONNX to circle conversion, or set `CM_PASS_DUMP=2` to see
157+
additional logs for circle rewrite.
158+
159+
```
160+
CM_PASS_DUMP=2 onnx2circle input.onnx output.circle
161+
```
162+
163+
You can give `-debug` option to see general MLIR logs or `-debug-only=o2c`
164+
option to see only logs from onnx2circle.
165+
166+
```
167+
onnx2circle -debug-only=o2c input.onnx output.circle
168+
```
169+
170+
## TensorFlow source code
171+
172+
Some source codes are referenced from TensorFlow and the file path is added to
173+
inside our source.
174+
175+
Current codes are from `v2.12.1` tag.

0 commit comments

Comments
 (0)