Skip to content

Commit 73317ea

Browse files
trzecieutrzecieu
committed
Fix for #45 #46 and more
* set 1.38.33 as ignored due probelm with dependencies * added python3 and python3-pip to slim image * mitigate new build system introduced by upstream project * temporarely removed possibility to compile incoming + master
1 parent 785603d commit 73317ea

File tree

6 files changed

+108
-19
lines changed

6 files changed

+108
-19
lines changed

‎build

+15-5
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ LOG_COMPILATION = "build.log"
4747
SDK_IGNORED = [
4848
"1.38.7", # malformed due an incompatibility with EMSDK release
4949
"1.38.17", # ignored due an SDK error
50+
"1.38.33", # broken dependency between emscipten<>emscripten-releases
5051
]
5152

5253
TEST_CASES = {
@@ -320,7 +321,8 @@ class DockerHelper:
320321
def decompose_image_id(image_id):
321322
print(image_id)
322323
project, tag = image_id.split(":")
323-
return (project, tag)
324+
version = re.match(r'.+(\d+\.\d+\.\d+).+', tag).groups()[0]
325+
return (project, tag, version)
324326

325327
@staticmethod
326328
def push_image(image, clean=False):
@@ -420,7 +422,9 @@ def get_builds(tags, update=False, branches=None, releases=False):
420422

421423
def compile_image(image_id, emsdk_cs, no_cache = False):
422424
t_start = datetime.datetime.now()
423-
docker_project, emscripten_sdk = DockerHelper.decompose_image_id(image_id)
425+
print("---")
426+
print(image_id)
427+
docker_project, emscripten_sdk, emscripten_version = DockerHelper.decompose_image_id(image_id)
424428
log("[INFO] Start compiling [{i}]".format(i=image_id))
425429
with open("build_{}.log".format(image_id.replace("/","_")), "w") as f:
426430
cmd = []
@@ -431,7 +435,8 @@ def compile_image(image_id, emsdk_cs, no_cache = False):
431435
cmd += ["--no-cache"] if no_cache else []
432436
cmd += [
433437
"-t", image_id,
434-
"--build-arg", "EMSCRIPTEN_SDK=" + emscripten_sdk,
438+
# "--build-arg", "EMSCRIPTEN_SDK=" + emscripten_sdk,
439+
"--build-arg", "EMSCRIPTEN_VERSION=" + emscripten_version,
435440
"--build-arg", "EMSDK_CHANGESET=" + emsdk_cs,
436441
"docker/" + docker_project
437442
]
@@ -570,6 +575,11 @@ def create_compilation_sets(variants):
570575

571576
def compile(args):
572577
"""Build images, and push tags to the queue"""
578+
579+
assert(not args.branches) # not supported ATM
580+
assert(not args.incoming) # not supported ATM
581+
assert(not args.master) # not supported ATM
582+
573583
variants = get_variants_to_compile(args)
574584
sets = create_compilation_sets(variants)
575585

@@ -586,7 +596,6 @@ def compile(args):
586596
for i, v in enumerate(s.variants):
587597
image_id = v.full_name
588598
dashboard.set_task(image_id, i + 1, len(s.variants))
589-
590599
if compile_image(image_id, args.emsdk_cs, no_cache = args.no_cache) and EmscriptenTester.test_image(image_id) :
591600
log("{} - SUCCESS".format(image_id), True)
592601
else:
@@ -669,6 +678,7 @@ if __name__ == "__main__":
669678
parser_build.add_argument("--releases", action="store_true", help="Update released SDKs")
670679
parser_build.add_argument("--fast-fail", action="store_true", help="Stops queue after first failure")
671680
parser_build.add_argument("--emsdk-cs", default="master", help="Explicitly use given branch/changeset of juj/emsdk")
681+
parser_build.add_argument("--no-cache", action="store_true", help="Sets --no-cache for docker build command")
672682

673683
parser_test = subparsers.add_parser("test", help="Test given tag(s) with Emscripten and WebAssembly compatibility")
674684
parser_test.set_defaults(function=test)
@@ -678,7 +688,6 @@ if __name__ == "__main__":
678688

679689
parser_push = subparsers.add_parser("push", help="Runs a service what will push created images")
680690
parser_push.add_argument("--clean", action="store_true", help="Remove pushed images")
681-
parser_push.add_argument("--no-cache", action="store_true", help="Sets --no-cache for docker build command")
682691
parser_push.add_argument("--no-block", action="store_true", help="Don't wait if queue is empty")
683692
parser_push.set_defaults(function=push)
684693

@@ -687,4 +696,5 @@ if __name__ == "__main__":
687696
parser_set_latest.set_defaults(function=set_latest)
688697

689698
args = parser.parse_args()
699+
690700
args.function(args)

‎docker/trzeci/emscripten-slim/Dockerfile

+75-7
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ FROM debian:stretch AS stage_build
33
# ------------------------------ ABOUT THIS IMAGE -----------------------------
44
# This Dockerfile has two major sections:
55
# * STAGE BUILD: Which uses required tools to build a static version of Emscripten SDK
6-
# * STAGE DEPLOY: Which copies folder of Emscripten (/emsd_portable) from previous stage, and installs very based tools to make Emscripten work.
6+
# * STAGE DEPLOY: Which copies folder of Emscripten (/emsdk_portable) from previous stage, and installs very based tools to make Emscripten work.
77
#
88
# Compiled Emscripten SDK meant to be ready to go out of the shelf. That is `/emsdk_portable`:
99
# - contains every required part of Emscripten SDK
@@ -19,8 +19,8 @@ FROM debian:stretch AS stage_build
1919
# ------------------------------------------------------------------------------
2020
# -------------------------------- STAGE BUILD --------------------------------
2121
# ------------------------------------------------------------------------------
22-
23-
ARG EMSCRIPTEN_SDK=sdk-tag-1.37.16-64bit
22+
ARG EMSCRIPTEN_VERSION=1.38.33
23+
ARG EMSCRIPTEN_SDK=sdk-tag-${EMSCRIPTEN_VERSION}-64bit
2424
ARG EMSDK_CHANGESET=master
2525

2626
# ------------------------------------------------------------------------------
@@ -44,7 +44,6 @@ RUN echo "## Update and install packages" \
4444
python-pip \
4545
&& echo "## Done"
4646

47-
4847
RUN echo "## Installing CMake" \
4948
&& wget https://cmake.org/files/v3.6/cmake-3.6.3-Linux-x86_64.sh -q \
5049
&& mkdir /opt/cmake \
@@ -55,18 +54,81 @@ RUN echo "## Installing CMake" \
5554
RUN echo "## Get EMSDK" \
5655
&& git clone https://github.com/emscripten-core/emsdk.git ${EMSDK} && cd ${EMSDK} && git reset --hard ${EMSDK_CHANGESET} \
5756
\
58-
# Update emscripten-tags.txt file manually with request version, this will tell emsdk that wanted version to compile is available
59-
&& echo ${EMSCRIPTEN_SDK} | sed 's/.\+-\([0-9]\+\.[0-9]\+\.[0-9]\+\).\+/\1/g' > emscripten-tags.txt \
57+
&& ./emsdk.py update-tags \
58+
&& releases_revision=`cat ./emscripten-releases-tags.txt | sed -n "s/.*${EMSCRIPTEN_VERSION}.*\"\(.*\)\".*/\1/p"` \
59+
&& ( cd releases && git reset --hard ${releases_revision} ) \
6060
&& echo "## Done"
6161

62+
# ------------------------------------------------------------------------------
63+
# This sections aims to mitigate new waterfall build system that has been used in emscripten
64+
# and the fact that it's not possible anymore to build new SDKs from sources
65+
66+
RUN echo "## Compose Fastcomp" \
67+
&& cd ${EMSDK} && mkdir -p zips \
68+
\
69+
&& repository=https://github.com/emscripten-core/emscripten-fastcomp \
70+
&& revision=`cat ./releases/DEPS | sed -n "s/.*fastcomp_revision.*'\(.*\)'.*/\1/p"` \
71+
&& touch ./zips/llvm-e${EMSCRIPTEN_VERSION}.tar.gz \
72+
&& git clone $repository clang/tag-e${EMSCRIPTEN_VERSION}/src \
73+
&& (cd clang/tag-e${EMSCRIPTEN_VERSION}/src && git reset --hard ${revision}) \
74+
\
75+
&& echo "## Done"
76+
77+
RUN echo "## Compose Fastcomp - Clang" \
78+
&& cd ${EMSDK} && mkdir -p zips \
79+
\
80+
&& repository=https://github.com/emscripten-core/emscripten-fastcomp-clang \
81+
&& revision=`cat ./releases/DEPS | sed -n "s/.*fastcomp_clang_revision.*'\(.*\)'.*/\1/p"` \
82+
&& touch ./zips/clang-e${EMSCRIPTEN_VERSION}.tar.gz \
83+
&& git clone $repository clang/tag-e${EMSCRIPTEN_VERSION}/src/tools/clang \
84+
&& (cd clang/tag-e${EMSCRIPTEN_VERSION}/src/tools/clang && git reset --hard ${revision}) \
85+
\
86+
&& echo "## Done"
87+
88+
RUN echo "## Compose Binaryen" \
89+
&& cd ${EMSDK} && mkdir -p zips \
90+
\
91+
&& repository=https://github.com/WebAssembly/binaryen \
92+
&& revision=`cat ./releases/DEPS | sed -n "s/.*binaryen_revision.*'\(.*\)'.*/\1/p"` \
93+
&& touch ./zips/binaryen-e${EMSCRIPTEN_VERSION}.tar.gz \
94+
&& git clone $repository binaryen/tag-${EMSCRIPTEN_VERSION} \
95+
&& (cd binaryen/tag-${EMSCRIPTEN_VERSION} && git reset --hard ${revision}) \
96+
\
97+
&& echo "## Done"
98+
99+
RUN echo "## Compose Emscripten" \
100+
&& cd ${EMSDK} && mkdir -p zips \
101+
\
102+
&& repository=https://github.com/emscripten-core/emscripten \
103+
&& revision=`cat ./releases/DEPS | sed -n "s/.*emscripten_revision.*'\(.*\)'.*/\1/p"` \
104+
&& touch ./zips/emscripten-e${EMSCRIPTEN_VERSION}.tar.gz \
105+
&& git clone $repository emscripten/tag-${EMSCRIPTEN_VERSION} \
106+
&& (cd emscripten/tag-${EMSCRIPTEN_VERSION} && git reset --hard ${revision}) \
107+
\
108+
&& echo "## Done"
109+
110+
# patch emsdk in order to install all required tools from repositories
111+
# fix for: #45 #46
112+
RUN echo "## Patch EMSDK" \
113+
&& cd ${EMSDK} \
114+
&& printf "\n${EMSCRIPTEN_VERSION}" > legacy-emscripten-tags.txt \
115+
&& printf "\n${EMSCRIPTEN_VERSION}" > legacy-binaryen-tags.txt \
116+
&& sed -i 's/download_even_if_exists = True/download_even_if_exists = False # a hack to install binaryen from repo/' ./emsdk.py \
117+
&& sed -i "s/.*${EMSCRIPTEN_VERSION}.*//g" ./emscripten-releases-tags.txt \
118+
&& echo "\"${EMSCRIPTEN_VERSION}\"" > ./emscripten/tag-${EMSCRIPTEN_VERSION}/emscripten-version.txt \
119+
&& echo "## Done"
120+
121+
# / end of work-around section
122+
# ------------------------------------------------------------------------------
123+
62124
RUN echo "## Compile Emscripten" \
63125
&& cd ${EMSDK} \
64126
\
65127
&& ./emsdk install node-8.9.1-64bit > /dev/null \
66128
# Compile llvm with dynamic libs support
67129
# This will create local shared objects that are used on all LLVM tool.
68130
# It significantly reduces final size of image.
69-
&& LLVM_CMAKE_ARGS=-DLLVM_LINK_LLVM_DYLIB=ON,-DLLVM_LINK_LLVM_DYLIB=ON \
131+
&& LLVM_CMAKE_ARGS=-DLLVM_LINK_LLVM_DYLIB=ON \
70132
./emsdk install --build=MinSizeRel ${EMSCRIPTEN_SDK} \
71133
\
72134
&& echo "## Done"
@@ -282,6 +344,8 @@ RUN echo "## Update and install packages" \
282344
ca-certificates \
283345
python \
284346
python-pip \
347+
python3 \
348+
python3-pip \
285349
\
286350
# Standard Cleanup on Debian images
287351
&& apt-get -y clean \
@@ -335,6 +399,8 @@ RUN echo "## Internal Testing of image (activated)" \
335399
&& npm --version \
336400
&& python --version \
337401
&& pip --version \
402+
&& python3 --version \
403+
&& pip3 --version \
338404
\
339405
&& em++ --version \
340406
&& emcc --version \
@@ -356,6 +422,8 @@ RUN echo "## Internal Testing of image (no activation)" \
356422
&& npm --version \
357423
&& python --version \
358424
&& pip --version \
425+
&& python3 --version \
426+
&& pip3 --version \
359427
\
360428
&& em++ --version \
361429
&& emcc --version \

‎docker/trzeci/emscripten-ubuntu/Dockerfile

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
ARG EMSCRIPTEN_SDK=sdk-tag-1.38.25-64bit
2-
FROM trzeci/emscripten:${EMSCRIPTEN_SDK} as emscripten_base
1+
ARG EMSCRIPTEN_VERSION=1.38.33
2+
FROM trzeci/emscripten:sdk-tag-${EMSCRIPTEN_VERSION}-64bit as emscripten_base
33

44
# ------------------------------------------------------------------------------
55
# Entries in this section is required to move Emscripten SDK to another base image

‎docker/trzeci/emscripten/Dockerfile

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
ARG EMSCRIPTEN_SDK=sdk-tag-1.37.16-64bit
2-
FROM trzeci/emscripten-slim:${EMSCRIPTEN_SDK}
1+
ARG EMSCRIPTEN_VERSION=1.38.33
2+
FROM trzeci/emscripten-slim:sdk-tag-${EMSCRIPTEN_VERSION}-64bit
33

44
# ------------------------------ ABOUT THIS IMAGE -----------------------------
55
# This image shows one way of extending base image which is `emscripten-slim` by

‎emscripten-slim.md

+7-1
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,12 @@ docker inspect --format '{{ index .Config.Labels "org.label-schema.vcs-ref"}}'
190190
## History
191191
<sub>(Please note that following history refers only to the history of this Docker Image and how it was build / what includes. For release notes of emscripten, please follow https://emscripten.org)</sub>
192192

193-
* **1.38.33**: [#44](https://github.com/trzecieu/emscripten-docker/issues/44) Significant refactoring of base image emscripten-slim. Please visit issue, to get extended context and motivation.
193+
* **1.38.34**: Cumulative change
194+
[#45](https://github.com/trzecieu/emscripten-docker/issues/45)[#46](https://github.com/trzecieu/emscripten-docker/issues/46) Adopt image to new Emscripten SDK
195+
* Starting from 1.38.33 Emscripten has switched project build system to waterfall, which has introduced some complication to building the image. The main problem was that no longer was possible to compile Emscripten from sources, with making sure that we have exactly the same components like Emscripten was released. For the instance WebAssembly/binaryen repo wasn't tagged with Emscripten version anymore.
196+
* Added python3, python3-pip package as requested in [WebAssembly/binaryen#2281](https://github.com/WebAssembly/binaryen/pull/2281).
197+
198+
[#44](https://github.com/trzecieu/emscripten-docker/issues/44) Significant refactoring of base image emscripten-slim. Please visit issue, to get extended context and motivation.
194199
* Improvements:
195200
* `/emsdk_portable` is fully moveable folder that can be used as a `COPY --from` source of multi stage build
196201
* `/emsdk_portable/dockerfiles` contains Dockerfile sources used to compile a particular image - so that it should be fairly easy to replicate and inspect content of images
@@ -206,6 +211,7 @@ docker inspect --format '{{ index .Config.Labels "org.label-schema.vcs-ref"}}'
206211
* Image no longer preserves folder structure between versions, some tool might be placed in different location between versions
207212
* Instead it creates symbolic links in fixed locations that match old structure
208213
* `nodejs` is no longer symlinked (`node` should be used instead)
214+
* **1.38.33**: Version ignored due problem with dependencies
209215
* **1.38.26** [#36](https://github.com/trzecieu/emscripten-docker/issues/36) - Reduce image size from 330MB to 189MB
210216
* **1.38.20** [#34](https://github.com/trzecieu/emscripten-docker/issues/34) - Fix error when `emcc` tries to read internal documentation
211217
* **1.38.17** Version ignored due problems with [Emscripten]

‎emscripten.md

+7-2
Original file line numberDiff line numberDiff line change
@@ -188,12 +188,16 @@ docker inspect --format '{{ index .Config.Labels "org.label-schema.vcs-ref"}}'
188188
## History
189189
<sub>(Please note that following history refers only to the history of this Docker Image and how it was build / what includes. For release notes of emscripten, please follow https://emscripten.org)</sub>
190190

191-
* **1.38.33**: [#44](https://github.com/trzecieu/emscripten-docker/issues/44) Significant refactoring of base image emscripten-slim. Please visit issue, to get extended context and motivation.
191+
* **1.38.34**: Cumulative change
192+
[#45](https://github.com/trzecieu/emscripten-docker/issues/45)[#46](https://github.com/trzecieu/emscripten-docker/issues/46) Adopt image to new Emscripten SDK
193+
* Starting from 1.38.33 Emscripten has switched project build system to waterfall, which has introduced some complication to building the image. The main problem was that no longer was possible to compile Emscripten from sources, with making sure that we have exactly the same components like Emscripten was released. For the instance WebAssembly/binaryen repo wasn't tagged with Emscripten version anymore.
194+
195+
[#44](https://github.com/trzecieu/emscripten-docker/issues/44) Significant refactoring of base image emscripten-slim. Please visit issue, to get extended context and motivation.
192196
* Improvements:
193197
* `/emsdk_portable` is fully moveable folder that can be used as a `COPY --from` source of multi stage build
194198
* `/emsdk_portable/dockerfiles` contains Dockerfile sources used to compile a particular image - so that it should be fairly easy to replicate and inspect content of images
195199
* `emsdk` should be fully functional tool now, so that can be used for upgrading bundled emscripten SDK or to install extra tools
196-
* Even further size optimization (From ~190MB to ~160MB) by:
200+
* Even further size optimization for slim image (From ~190MB to ~160MB) by:
197201
* stripping out symbols from node.js and emscripten-clang tools
198202
* change base image to `debiang:stretch-slim`
199203
* remove non-essential files from emscripten folder
@@ -204,6 +208,7 @@ docker inspect --format '{{ index .Config.Labels "org.label-schema.vcs-ref"}}'
204208
* Image no longer preserves folder structure between versions, some tool might be placed in different location between versions
205209
* Instead it creates symbolic links in fixed locations that match old structure
206210
* `nodejs` is no longer symlinked (`node` should be used instead)
211+
* **1.38.33**: Version ignored due problem with dependencies
207212
* **1.38.30**: [#40](https://github.com/trzecieu/emscripten-docker/issues/40) Fixed image compilation problem caused by JRE backport package
208213
* **1.38.22**: [#35](https://github.com/trzecieu/emscripten-docker/issues/35) upgrade to `cmake` 3.12.2
209214
* **1.38.17**: Version ignored due problems with [Emscripten]

0 commit comments

Comments
 (0)