Skip to content

Commit

Permalink
spatialite 5.1
Browse files Browse the repository at this point in the history
  • Loading branch information
@jvail
jvail committed Nov 26, 2023
1 parent b44f6a5 commit 902c374
Show file tree
Hide file tree
Showing 37 changed files with 6,540 additions and 5,579 deletions.
109 changes: 53 additions & 56 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@ BUILD_DIR = $(PWD)/src/build
BC_DIR = $(BUILD_DIR)/bc
PREFIX = --prefix=$(BC_DIR)

SQLITE_VERSION = 3390400
GEOS_VERSION = 3.9.0
PROJ_VERSION = 9.1.0
SQLITE_VERSION = 3440100
GEOS_VERSION = 3.12.0
PROJ_VERSION = 9.2.1
RTTOPO_VERSION = 1.1.0

ZLIB_VERSION = 1.2.13
ZLIB_VERSION = 1.3
XML2_VERSION = 2.9.12
ICONV_VERSION = 1.16
ICONV_VERSION = 1.17

SPATIALITE_SRC = src/spatialite
SQLITE_SRC = $(BUILD_DIR)/sqlite-autoconf-$(SQLITE_VERSION)
Expand All @@ -22,13 +22,23 @@ XML2_SRC = $(BUILD_DIR)/libxml2-$(XML2_VERSION)
ICONV_SRC = $(BUILD_DIR)/libiconv-$(ICONV_VERSION)
SQLEAN_EXT_SRC = src/sqlean/src

DIST_FLAGS :=
DIST_FLAGS += -s EXPORT_ES6=1
DIST_FLAGS += -s USE_ES6_IMPORT_META=0
DIST_FLAGS += -s MODULARIZE=1
DIST_FLAGS += -s EXPORT_NAME="spl"
DIST_FLAGS += -s ENVIRONMENT="node,worker"
DIST_FLAGS += -s WASM_ASYNC_COMPILATION=0
DIST_WORKER_FLAGS :=
DIST_WORKER_FLAGS += -s EXPORT_ES6=1
DIST_WORKER_FLAGS += -s USE_ES6_IMPORT_META=0
DIST_WORKER_FLAGS += -s MODULARIZE=1
DIST_WORKER_FLAGS += -s EXPORT_NAME="spl"
DIST_WORKER_FLAGS += -s ENVIRONMENT="worker"
DIST_WORKER_FLAGS += -s WASM_ASYNC_COMPILATION=0
DIST_WORKER_FLAGS += --memory-init-file 0

DIST_NODE_FLAGS :=
DIST_NODE_FLAGS += -s EXPORT_ES6=1
DIST_NODE_FLAGS += -s USE_ES6_IMPORT_META=1
DIST_NODE_FLAGS += -s MODULARIZE=1
DIST_NODE_FLAGS += -s EXPORT_NAME="spl"
DIST_NODE_FLAGS += -s ENVIRONMENT="node"
DIST_NODE_FLAGS += -s WASM_ASYNC_COMPILATION=0
DIST_NODE_FLAGS += --memory-init-file 0

ELD_FLAGS += -s INITIAL_MEMORY=64MB
ELD_FLAGS += -s ALLOW_MEMORY_GROWTH=1
Expand All @@ -40,7 +50,7 @@ ELD_FLAGS += -lworkerfs.js

EMX_FLAGS :=
EMX_FLAGS += -s DISABLE_EXCEPTION_CATCHING=0
EMX_FLAGS += --memory-init-file 0


EXPORTED_FUNCTIONS = -s EXPORTED_FUNCTIONS=@$(PWD)/src/exported_functions.json
EXPORTED_RUNTIME_METHODS = -s EXPORTED_RUNTIME_METHODS=@$(PWD)/src/exported_runtime_methods.json
Expand Down Expand Up @@ -83,7 +93,7 @@ zlib-src:

iconv-conf: iconv-src
cd $(ICONV_SRC); \
emconfigure ./configure $(PREFIX) --disable-shared;
emconfigure ./configure $(PREFIX) --disable-shared --host=none-none-none;

iconv: iconv-conf
cd $(ICONV_SRC); \
Expand All @@ -97,12 +107,12 @@ iconv-src:

sqlite-src:
cd $(BUILD_DIR); \
wget -nc https://www.sqlite.org/2022/sqlite-autoconf-$(SQLITE_VERSION).tar.gz; \
wget -nc https://www.sqlite.org/2023/sqlite-autoconf-$(SQLITE_VERSION).tar.gz; \
tar -xf sqlite-autoconf-$(SQLITE_VERSION).tar.gz;

sqlite-conf: sqlite-src
cd $(SQLITE_SRC); \
emconfigure ./configure $(PREFIX) \
emconfigure ./configure $(PREFIX) --host=none-none-none \
--disable-tcl --disable-shared --disable-editline --disable-readline --disable-load-extension;

sqlite: sqlite-conf
Expand All @@ -127,7 +137,7 @@ xml2-src:
xml2-conf: xml2-src
cd $(XML2_SRC); \
./autogen.sh $(PREFIX) --without-python; \
emconfigure ./configure $(PREFIX) --without-python --disable-shared --with-zlib CPPFLAGS="-DDEBUG_HTTP" Z_LIBS="-L$(BC_DIR)/lib -lz" Z_CFLAGS="-I$(BC_DIR)/include";
emconfigure ./configure $(PREFIX) --host=none-none-none --without-python --disable-shared --with-zlib CPPFLAGS="-DDEBUG_HTTP" Z_LIBS="-L$(BC_DIR)/lib -lz" Z_CFLAGS="-I$(BC_DIR)/include";

xml2: xml2-conf
cd $(XML2_SRC); \
Expand Down Expand Up @@ -166,33 +176,14 @@ geos-src:
wget -nc http://download.osgeo.org/geos/geos-$(GEOS_VERSION).tar.bz2; \
tar -xf geos-$(GEOS_VERSION).tar.bz2; \

# sed -i '/add_subdirectory(doc)/d' $(GEOS_SRC)/CMakeLists.txt;

# geos: geos-src
# cd $(GEOS_SRC) && mkdir -p build && cd build; \
# emcmake cmake -DCMAKE_CXX_FLAGS="$(EMX_FLAGS)" \
# -DDISABLE_GEOS_INLINE=ON -DBUILD_TESTING=OFF -DBUILD_BENCHMARKS=OFF -DBUILD_SHARED_LIBS=OFF \
# -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$(BC_DIR) ..; \
# emmake make -j4; \
# emmake make install;

geos-conf: geos-src
cd $(GEOS_SRC); \
emconfigure ./configure $(PREFIX) --disable-inline --disable-shared;

# sed -i '/doc/d' $(GEOS_SRC)/Makefile;
# sed -i '/tests/d' $(GEOS_SRC)/Makefile;
# sed -i '/benchmarks/d' $(GEOS_SRC)/Makefile;
# sed -i '/tools/d' $(GEOS_SRC)/Makefile;

geos: geos-conf
cd $(GEOS_SRC); \
emmake make -j4 \
CPPFLAGS="$(EMX_FLAGS)"; \
geos: geos-src
cd $(GEOS_SRC) && mkdir -p build && cd build; \
emcmake cmake -DCMAKE_CXX_FLAGS="$(EMX_FLAGS)" \
-DBUILD_TESTING=OFF -DBUILD_BENCHMARKS=OFF -DBUILD_SHARED_LIBS=OFF \
-DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$(BC_DIR) ..; \
emmake make -j4; \
emmake make install;

# cd $(GEOS_SRC)/tools && emmake make install geos-config;

geos-clean:
cd $(GEOS_SRC) && make clean;

Expand All @@ -204,7 +195,7 @@ rttopo-src:
rttopo-conf: rttopo-src
cd $(RTTOPO_SRC); \
./autogen.sh; \
emconfigure ./configure $(PREFIX) --disable-shared --with-geosconfig="$(BC_DIR)/bin/geos-config";
emconfigure ./configure $(PREFIX) --host=none-none-none --disable-shared --with-geosconfig="$(BC_DIR)/bin/geos-config";

rttopo: rttopo-conf
cd $(RTTOPO_SRC); \
Expand All @@ -214,9 +205,6 @@ rttopo: rttopo-conf
rttopo-clean:
cd $(RTTOPO_SRC) && make clean;

# LIBXML2_LIBS="-L$(BC_DIR)/lib"
# LIBXML2_CFLAGS="-I$(BC_DIR)/include/libxml2"

# remove some irrelevant tests that will fail (still some xls tests failing)
spatialite-conf:
sed -i '/check_extension/d' $(SPATIALITE_SRC)/test/Makefile.am;
Expand All @@ -227,14 +215,13 @@ spatialite-conf:
-rm $(SPATIALITE_SRC)/test/sql_stmt_security_tests/importxls*;
cd $(SPATIALITE_SRC); \
aclocal && automake; \
emconfigure ./configure $(PREFIX) \
emconfigure ./configure $(PREFIX) --host=none-none-none \
CFLAGS="$(EMX_FLAGS) -DENABLE_MINIZIP -UOMIT_PROJ -DPROJ_NEW -ULOADABLE_EXTENSION -I$(ZLIB_SRC)/contrib" \
CPPFLAGS="-I$(BC_DIR)/include" \
LDFLAGS="-L$(BC_DIR)/lib" \
PKG_CONFIG_LIBDIR="$(BC_DIR)/lib/pkgconfig" \
--with-geosconfig="$(BC_DIR)/bin/geos-config" \
--with-geosonlyreentrant \
--enable-rttopo \
--enable-geocallbacks \
--enable-geosadvanced \
--enable-geopackage \
--enable-gcp \
Expand All @@ -258,7 +245,7 @@ spatialite-clean:
extensions:
emcc -v -I$(SQLITE_SRC) -DSQLITE_CORE $(EMX_FLAGS) $(ELD_FLAGS) -c $(SQLEAN_EXT_SRC)/sqlite3-stats.c -o $(BC_DIR)/ex.o;


# requires node 21 to load fils without extension as mjs
tests:
cd $(SPATIALITE_SRC)/test; \
echo " \
Expand All @@ -273,19 +260,29 @@ tests:
};" > pre.js; \
mkdir -p proj && cp -f $(BC_DIR)/share/proj/* proj; \
emmake make check -j4 LOG_COMPILE="node" \
LDFLAGS="$(ELD_FLAGS) -s ENVIRONMENT=node -s FORCE_FILESYSTEM=1 -s ERROR_ON_UNDEFINED_SYMBOLS=0 -s EXPORTED_FUNCTIONS="[_main]" -liconv -lminizip" \
LDFLAGS="$(ELD_FLAGS) -lgeos -s WASM_ASYNC_COMPILATION=0 -s ENVIRONMENT=node -s EXPORT_ES6=1 -s USE_ES6_IMPORT_META=1 -s FORCE_FILESYSTEM=1 -s ERROR_ON_UNDEFINED_SYMBOLS=0 -s EXPORTED_FUNCTIONS="[_main]" -liconv -lminizip" \
CFLAGS="$(EMX_FLAGS) -D_GNU_SOURCE --pre-js pre.js";


.PHONY: spl
spl: src/pre.js
emcc -v $(EMX_FLAGS) $(ELD_FLAGS) $(DIST_FLAGS) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) \
.PHONY: worker
worker: src/pre.js
emcc -v $(EMX_FLAGS) $(ELD_FLAGS) $(DIST_WORKER_FLAGS) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) \
-lz -lminizip -liconv -lsqlite3 -lgeos_c -lgeos -lrttopo -lproj -L$(BC_DIR)/lib \
-s ERROR_ON_UNDEFINED_SYMBOLS=1 $(BC_DIR)/lib/libspatialite.a $(BC_DIR)/ex.o \
-I$(SQLITE_SRC) \
-I$(SPATIALITE_SRC)/src/headers \
-I$(SPATIALITE_SRC)/src/headers/spatialite \
src/spl.c --pre-js src/pre.js -o $(BUILD_DIR)/js/em-worker.js --embed-file $(BC_DIR)/share/proj_min.db@/proj_min.db;

.PHONY: node
node: src/pre.js
emcc -v $(EMX_FLAGS) $(ELD_FLAGS) $(DIST_NODE_FLAGS) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) \
-lz -lminizip -liconv -lsqlite3 -lgeos_c -lgeos -lrttopo -lproj -L$(BC_DIR)/lib \
-s ERROR_ON_UNDEFINED_SYMBOLS=0 $(BC_DIR)/lib/libspatialite.a $(BC_DIR)/ex.o \
-s ERROR_ON_UNDEFINED_SYMBOLS=1 $(BC_DIR)/lib/libspatialite.a $(BC_DIR)/ex.o \
-I$(SQLITE_SRC) \
-I$(SPATIALITE_SRC)/src/headers \
-I$(SPATIALITE_SRC)/src/headers/spatialite \
src/spl.c --pre-js src/pre.js -o $(BUILD_DIR)/js/spl.js --embed-file $(BC_DIR)/share/proj_min.db@/proj_min.db;
src/spl.c --pre-js src/pre.js -o $(BUILD_DIR)/js/index.js --embed-file $(BC_DIR)/share/proj_min.db@/proj_min.db;

clean:
rm -rf $(PWD)/dist/*;
Expand Down
33 changes: 6 additions & 27 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ SpatiaLite and friends - sqlite, geos, proj, rttopo - for node (sync API) and br

[Here is a list](doc/spatialite_functions.md) of supported SpatiaLite SQL functions and [a list](doc/extensions_functions.md) of available SQLite extension functions/modules.

Comments, bug reports and suggestions are welcome! spl.js will remain beta at least until the release of SpatiaLite 5.1.0.

## Install

```bash
Expand All @@ -17,24 +15,6 @@ The library for browsers bundles both the WebWorker script and the wasm file (~
A minimal proj.db including EPSG codes for lon/lat to "Web Mercator" and UTM is embeded. Other PROJ files and the complete proj.db are available from the `dist/proj` folder.
If you like to use the full proj.db instead you need to mount it to `/proj/proj.db` and set the path with spatialite's `PROJ_SetDatabasePath`.

[Apparently](https://github.com/jvail/spl.js/issues/1) typescript has no option to switch between `main` and `browser` entrypoints in `package.json`. For typescript & browser development you need to import spl.js (async version) as

```ts
import SPL from 'spl.js/dist/index'
```

or add this option to tsconfig.json

```json
{
"compilerOptions": {
"baseUrl": ".",
"paths": {
"spl.js": ["node_modules/spl.js/dist/index"]
}
}
}
```

## Code Examples

Expand All @@ -45,7 +25,7 @@ import SPL from 'spl.js';
const db = await SPL().then(spl => spl.db());

console.assert(
await db.exec('SELECT spatialite_version()').get.first === '5.0.1'
await db.exec('SELECT spatialite_version()').get.first === '5.1.0'
);

db.exec('SELECT ? AS hello', ['spatialite']).get.objs
Expand Down Expand Up @@ -148,7 +128,7 @@ If you are looking for more examples there are many snippets in the `test/node.j

## SPL

### `SPL`([`extensions`: [], `options`: {}]) -> `SPL`
### `SPL`([`options`: {}, `extensions`: []]) -> `SPL`

extensions: Browser only - see "Extensions API" section below.

Expand Down Expand Up @@ -247,8 +227,6 @@ With `sync` (browser only) all ArrayBuffers will be transfered without copying (

## Extensions API (Browser only)

*Experimental since Firefox does currently not support dynamic imports in Workers.*

Sometimes you want to run code inside the WebWorker. With this API you can supply additional functions to extend the `SPL` and `DB` APIs executed inside the WebWorker.

Example: https://jvail.github.io/spl.js/examples/extensions.html
Expand Down Expand Up @@ -277,7 +255,7 @@ const extensions = [
}
];

const spl = await SPL(extensions);
const spl = await SPL({}, extensions);
const db = await spl.db()
.read(`
CREATE TABLE hello (world);
Expand All @@ -286,12 +264,12 @@ const db = await spl.db()

console.assert(await db.tables().get.first === 'hello');
console.assert(await db.master('view').get.first === 'hello_view');
console.assert(await spl.spatialite_version() === '5.0.1');
console.assert(await spl.spatialite_version() === '5.1.0');
```

## Building and Testing

An activated, working emsdk environment (3.1.23) is required (https://emscripten.org/docs/tools_reference/emsdk.html). All dependencies except SpatiaLite & sqlean (git submodules) are fetched from the web. The git submodules needs to be initialized before running the build script.
An activated, working emsdk environment (3.1.49) is required (https://emscripten.org/docs/tools_reference/emsdk.html). All dependencies except SpatiaLite & sqlean (git submodules) are fetched from the web. The git submodules needs to be initialized before running the build script.

```bash
npm install && npm run build:all
Expand All @@ -304,6 +282,7 @@ npm run test:node && npm run test:firefox && npm run test:chrome
```

Running (the relevant) SpatiaLite test cases - this will take quite some time ... ~ 45 minutes or more.
Requires node >= v21 to run test cases without .mjs extension.

```bash
npm run test:em
Expand Down
54 changes: 0 additions & 54 deletions dist/index.d.ts
View file

This file was deleted.

55 changes: 53 additions & 2 deletions dist/index.js
View file

Large diffs are not rendered by default.

Loading

1 comment on commit 902c374

@alexbodn
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i see the inversion of SPL arguments i should do in my code

Please sign in to comment.