SQLite benchmark

.gitignore

@@ -9,3 +9,4 @@ seed_embeddings/preprocessed/*
 
 html
 latex
+sqlite3.ll

src/test-suite/CMakeLists.txt

@@ -1,3 +1,72 @@
+option(ENABLE_SQLITE "Enables SQLite Amalgamation Benchmark." ON)
+
+set(SQLITE_LLVM_IR
+    ${CMAKE_CURRENT_SOURCE_DIR}/PE-benchmarks-llfiles-llvm17/sqlite3.ll
+)
+
+# SQLite IR file is added along side other PE-benchmarks for this version
+if(NOT EXISTS ${SQLITE_LLVM_IR} AND ENABLE_SQLITE)
+    # The downloaded Zip file and source files directory are located at the
+    # build directory since its already gitignore anyways.
+    set(SQLITE_ZIP ${CMAKE_CURRENT_BINARY_DIR}/sqlite-amalgamation-3460000.zip)
+    set(SQLITE_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/sqlite-amalgamation-3460000)
+
+    # Some of the recommended compile flags. https://sqlite.org/amalgamation.html
+    # with the following flags the resulting LLVM IR file is about 500k lines
+    set(SQLITE_COMPILE_FLAGS
+        -DSQLITE_ENABLE_FTS3
+        -DSQLITE_ENABLE_FTS5
+        -DSQLITE_ENABLE_RTREE
+        -DSQLITE_ENABLE_DBSTAT_VTAB
+        -DSQLITE_ENABLE_RBU
+        -DSQLITE_ENABLE_SESSION
+    )
+
+    if(NOT EXISTS ${SQLITE_ZIP})
+        # SQLite Amalgamation download source: https://sqlite.org/download.html
+        message(STATUS "Downloading SQLite Amalgamation source zip file..")
+        file(
+        DOWNLOAD https://sqlite.org/2024/sqlite-amalgamation-3460000.zip
+        ${SQLITE_ZIP}
+        EXPECTED_HASH SHA3_256=1221eed70de626871912bfca144c00411f0c30d3c2b7935cff3963b63370ef7c
+        SHOW_PROGRESS
+        )
+    endif()
+
+    if(NOT EXISTS ${SQLITE_SOURCE})
+        message(STATUS
+            "Unzipping SQLite source files at: ${SQLITE_SOURCE}"
+        )
+        execute_process(
+            COMMAND unzip ${SQLITE_ZIP}
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+            RESULT_VARIABLE unzip_result
+        )
+
+        if(NOT unzip_result EQUAL 0)
+            message(FATAL_ERROR "Failed to extract SQLite zip file")
+        endif()
+    endif()
+
+    find_program(CLANG_BIN NAMES clang-17 clang)
+    if(NOT CLANG_BIN)
+        message(FATAL_ERROR "Clang not found. Please install it.")
+    endif()
+
+    execute_process(
+        COMMAND ${CLANG_BIN} -S -emit-llvm ${SQLITE_COMPILE_FLAGS}
+            -Xclang -disable-O0-optnone
+            -o ${SQLITE_LLVM_IR}
+            ${SQLITE_SOURCE}/sqlite3.c
+            RESULT_VARIABLE sqlite_ir_result
+    )
+
+    if(sqlite_ir_result EQUAL 0)
+        message(STATUS "SQLite benchmark was generated at: ${SQLITE_LLVM_IR}")
+    endif()
+endif()
+
+# sanity checks and lit configs
 configure_file(sanity_check.sh.cmake sanity_check.sh @ONLY)
 file(COPY PE-benchmarks-llfiles-llvm17 DESTINATION ./)
 file(COPY oracle DESTINATION ./)

src/test-suite/oracle/FA_llvm17_p/sqlite3.txt

@@ -0,0 +1 @@
+39389.228010	-5376.674414	-29021.478687	57311.999476	-2830.058969	-4903.917802	12531.761407	11238.552214	5837.613706	-75945.222073	-29524.528980	67012.326909	12066.566763	-3894.166052	-5155.274300	24880.990511	16379.301698	18119.941570	-78168.641123	6989.300529	12382.527812	-4140.765868	-22445.730997	-11017.107544	-2946.210374	-24964.022127	31784.060329	24619.920816	3814.250199	53991.318974	-15070.562586	30707.538848	43953.849707	-2381.574458	22950.984229	41734.144641	-10816.221965	-4810.135903	-22139.721114	-18213.911622	65218.353935	28923.175315	-4972.659720	25150.057632	-1473.603306	60679.434496	14821.739239	48281.290493	15413.638264	-21271.516911	17702.110667	29020.863980	-31464.468252	-12264.696262	-53225.776560	4551.504887	20417.821305	12091.340519	-24138.628052	28349.748690	-17150.800330	12706.377159	24665.181318	16145.634347	65299.241720	21431.810517	40498.453896	51106.281433	54662.926106	-40088.949771	13304.092762	7654.656533	14419.047690	1069.853817	48025.749385	-22396.022974	-42850.026510	-6809.055547	-521.476994	-64375.753597	-7862.747134	-5017.845790	81997.642405	-46507.572785	-7365.529276	1302.849919	25304.077449	58076.845632	-34610.699881	4141.103685	-51849.889573	-3722.191701	-9419.574215	79247.276286	9473.697538	19713.967952	-12465.907796	19244.744645	-3435.023306	13989.461256	6536.506768	-56492.493102	-1688.045743	-73137.700853	-29253.134499	-40407.956653	-933.743462	14279.813178	-57154.836462	557.310850	4767.098763	53947.134808	-84054.927478	-5960.262662	15618.135863	6821.010998	52393.624293	12726.230237	-20777.351490	9294.755970	1390.254836	17729.270299	23544.548798	-26589.157997	11527.892553	-2477.706692	2599.986765	10135.742292	17323.593387	18006.984562	-84040.870641	-13220.164514	23623.495802	-7462.858326	7428.499912	29600.097469	25956.154793	-22568.528168	22848.602044	-41683.667110	46822.194460	-63180.261022	-16181.937610	-23577.152061	-38918.014782	-27130.354619	40871.462790	45417.167782	45708.189488	-8005.036744	-9525.828909	-574.773616	-72716.434262	-90806.812223	10465.695405	-1704.417641	-3791.325645	11437.562356	-9734.447109	10740.288521	-43880.468152	14900.821761	34527.372778	-29568.994163	-26237.248706	-3183.854486	44694.863175	10560.089449	-18749.784008	-15393.422626	-7807.926404	-3875.797153	23007.442336	54734.520268	-18045.888336	-91677.309373	-81314.052401	-25529.987834	49294.973712	-25441.928778	-4832.642230	-21895.379659	-25103.559014	-31507.116894	-2475.733488	-39250.694689	12937.243692	-6512.779912	-17903.734550	-64470.347896	22094.732030	-9237.200299	-14267.520417	10192.716821	-47076.708383	10964.664559	30742.866710	4235.252363	1191.728237	10027.339188	4415.971911	-38850.183036	-29519.167851	20492.481769	-23659.175199	52721.378257	42300.583697	2340.791768	-29988.064784	-19580.261322	-49547.471529	33755.555148	-48905.776848	35422.181670	8024.147986	-27027.556318	33921.390322	-44944.517675	-7937.652834	-29487.218742	26365.878881	6197.248727	-71044.945394	70024.038121	51335.191099	-42336.803306	-3397.896498	-9550.309290	23323.897056	11667.371271	39492.749377	-43755.789585	-36460.205077	14610.025897	151.649603	-1885.533970	-2971.605785	14692.409094	-12728.623378	-2336.804940	-48256.526733	-15118.845499	47250.475424	26403.952496	-6263.492780	1761.811526	13312.365912	-25423.677786	12047.566917	49352.438572	30559.417573	12879.122901	-45558.117241	-102363.066526	21858.284461	-27296.611234	-9673.486010	6833.930802	-27911.443354	-18580.310923	16977.709576	-28889.467960	48066.249561	-17385.422090	-33583.855529	46193.509969	-69949.758441	-54972.692493	-46011.694699	-13363.262082	11685.615563	-4471.066728	22318.769605	-21219.305738	9400.389388	-7905.007664	19022.545764	-19689.155351	-29574.226282	12494.780265	28997.563923	45806.675927	7127.876997	-14628.763346	57523.118124	9747.445151	-12639.944425	-14974.659152	-5337.028753	-25489.188876	8811.413006	-66892.993020	-10358.495763	4699.510652	33516.701552	-40010.545447	22338.301792	70.999160	17700.864559	10359.928443	

src/test-suite/oracle/SYM_llvm17_p/sqlite3.txt

@@ -0,0 +1 @@
+16914.214172	1840.963612	-8056.768579	21579.305453	-906.262205	-14727.750711	4137.897728	12411.245494	-6586.577729	-37492.964734	-23503.579107	24019.665282	3178.140591	-4723.500753	5407.842244	7912.921167	-3458.007608	3013.286848	-43144.929418	15115.546246	13531.053632	4908.258806	-5820.302126	-7871.660645	3889.551273	-10608.906240	1448.299046	10127.834420	1014.800933	22875.036123	-11217.396606	7132.652507	18359.679961	-541.432431	9754.327113	19447.189792	-5458.297947	1001.803515	1260.266054	429.515246	30198.629348	18262.393044	-3520.028532	10885.129589	9038.867032	23509.461660	-2190.924487	20668.772705	-4464.237518	-15003.251139	10138.776080	7383.103845	-17749.750384	-2971.023222	-14480.273423	9624.178060	19172.138324	8872.303230	-11147.777716	11520.362299	-11789.698651	-2464.035909	7416.324752	381.487262	27537.998172	14634.787225	15946.290784	14708.178722	30142.986762	-11951.156505	-2847.544498	13036.279684	11946.873288	-2281.002892	14178.835480	-9445.563955	-21959.072204	-11427.626110	14.950048	-38164.969681	-15326.407916	-604.005724	40969.720254	-22921.026771	1021.245459	6482.292367	8335.949495	22709.037213	-15671.772052	-634.916992	-16388.880672	-88.372465	-10411.435811	38141.528995	1444.237535	7092.407774	-3015.810814	9673.987822	10480.803898	17367.615576	-5099.230566	-15713.135321	-16277.581293	-37081.163429	-8470.125739	-8772.651484	3261.668365	7589.841179	-31083.117699	-6787.744450	7673.834177	27727.294920	-34264.453388	-1024.019126	18412.358452	9384.182441	16855.866440	7579.575966	-20625.224866	4570.472021	1587.196202	1714.091642	13996.430750	-8729.059297	-1445.531903	-5345.869665	6737.841282	747.071812	5712.435997	1316.571824	-37393.893115	-11137.285625	3457.923217	-7597.644020	934.439596	4310.015852	8189.372290	-13761.423625	12699.496546	-14782.421306	18183.527744	-33691.703352	-517.262789	-11239.399834	-19974.966452	-3504.060581	16885.975472	25761.155805	14053.165475	-6681.317758	-6153.418469	-7325.079859	-30356.658229	-34656.871696	6283.765911	-1758.163001	8988.464773	-1522.551626	-1673.995728	5009.828211	-9656.368676	1426.665633	9335.348034	-11649.556808	-1566.318860	-6295.734684	31544.197891	661.649733	-4830.674434	-14478.650575	-8427.805505	-545.973874	2502.690463	15002.196104	-6300.676337	-33845.085246	-34194.679896	-18256.095681	19120.451916	-4796.204741	-1357.198324	-13606.642437	-3021.046722	-18445.044067	3332.247063	-17445.313795	6821.379552	-1561.129509	-13353.268278	-23115.474322	472.936692	-1898.996769	-9484.504976	-5017.053747	-15333.591479	-5749.949470	18458.335412	4047.862621	8084.542286	3028.814311	1129.188741	-21428.210033	-19353.038396	5786.410664	-7532.951615	21430.806942	19577.147353	459.479687	-18445.971763	-4433.705096	-27620.540913	19828.835893	-17033.978568	4474.063320	-6138.081727	-9289.859715	29276.565507	-13631.410108	-7187.016573	-18083.697502	8781.137185	3854.077762	-34757.682776	32445.021522	20843.803376	-11541.272980	-4382.951260	-1828.406638	2398.068724	18359.047156	21539.259133	-15484.985073	-13341.607581	3250.685126	-6791.923469	-1860.182208	-395.840937	16642.290996	-16803.371117	11532.507834	-15286.484050	-9243.681272	12537.733834	16148.825960	493.864148	8958.987453	-833.191497	-2730.725876	8574.123858	17163.311834	13874.951473	-6250.890749	-19190.295017	-44012.399335	13558.447731	-8566.129714	-6785.466744	-403.105960	-9429.032390	-19264.410796	3993.546581	-5802.231716	24354.598056	802.353023	-11928.318532	16617.764262	-27415.844372	-24580.951602	-13247.186538	2753.917039	-291.838695	-5530.368520	7072.636033	-7053.490037	6451.836519	9084.519734	856.179694	1720.038980	-17071.204665	-10241.571031	-236.670560	18695.744940	8736.999602	-4602.100388	24403.244235	5754.482284	-10343.410231	-6480.548159	858.463332	31.853096	-2285.052285	-20497.891846	-9698.522648	-9221.781745	8000.340915	-9367.694417	14972.296732	1337.978384	1709.723559	9941.290745	

src/test-suite/sanity_check.sh.cmake

@@ -1,5 +1,6 @@
 
 BUILD=@LIB_LOC@
+ENABLE_SQLITE=@ENABLE_SQLITE@
 EncodingType=$1
 
 SO_FILE=
@@ -43,13 +44,26 @@ perform_vector_comparison() {
     ORIG_FILE=oracle/${EncodingType}_${SEED_VERSION}_${FILE_PREFIX}/ir2vec.txt
     VIR_FILE=ir2vec_${FILE_PREFIX}.txt
 
+
+    # SQLite specific variables.
+    if [[ "$ENABLE_SQLITE" == "ON" ]]; then
+        SQLITE_VIR=sqlite3_${FILE_PREFIX}.txt
+        SQLITE_INPUT=./PE-benchmarks-llfiles-${SEED_VERSION}/sqlite3.ll
+        SQLITE_ORIG=oracle/${EncodingType}_${SEED_VERSION}_${FILE_PREFIX}/sqlite3.txt
+    fi
+
     # if file prefix is p or f, run the first while loop, else, run the second while loop
 
     if [[ "$FILE_PREFIX" == "p" || "$FILE_PREFIX" == "f" ]]; then
         while IFS= read -r d; do
             ${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${VIR_FILE} ${d} &> /dev/null
         done < index-${SEED_VERSION}.files
         wait
+
+        # SQLITE is currently only tested against the program (p) level
+        if [[ "$ENABLE_SQLITE" == "ON" && "$FILE_PREFIX" == "p" ]]; then
+            ${IR2VEC_PATH} -${PASS} -level ${LEVEL} -o ${SQLITE_VIR} ${SQLITE_INPUT} &> /dev/null
+        fi
     else
         while IFS= read -r d_on
         do
@@ -78,6 +92,25 @@ perform_vector_comparison() {
             echo -e "$(tput bold)${RED}[Error] No embeddings are generated.${NC}"
             exit 1
         fi
+
+        # SQLite tests only if its enabled
+        if [[ "$ENABLE_SQLITE" == "ON" ]]; then
+            if [[ ! -e "$SQLITE_VIR" ]]; then
+                echo -e "$(tput bold)${RED}[Error] No embeddings are generated for SQLite benchmark.${NC}"
+                exit 1
+            fi
+            mv ${SQLITE_VIR} ${TEMP}/
+
+            d=$(diff <(sed -e 's/^ *#[0-9]* *//g' ${SQLITE_ORIG}) <(sed -e 's/^ *#[0-9]* *//g' ${TEMP}/${SQLITE_VIR}))
+
+            if [ "$d" == "" ]; then
+                echo -e "${GREEN}${BOLD}[Test Passed] SQLite Benchmark Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Identical.${NC}"
+            else
+                echo -e "$(tput bold)${RED}[Test Failed] SQLite Benchmark Vectors of Oracle and Current version of ${FILE_PREFIX}-level are Different.${NC}"
+                exit 1
+            fi
+        fi
+
     else
         if ls *${VIR_FILE} 1> /dev/null 2>&1
         then