From c9ef6bacdf49dafd9709e44a70bcde17480046c0 Mon Sep 17 00:00:00 2001 From: lostbean Date: Fri, 24 Nov 2023 11:26:20 -0300 Subject: [PATCH 1/3] move local defined nix packages to files and use buildBazelPackage --- flake.nix | 41 ++++++------------------------------ nix-pkgs/grpc-tools-node.nix | 14 ++++++++++++ nix-pkgs/protoc-gen-ts.nix | 31 +++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 35 deletions(-) create mode 100644 nix-pkgs/grpc-tools-node.nix create mode 100644 nix-pkgs/protoc-gen-ts.nix diff --git a/flake.nix b/flake.nix index d4cf271ab5..99e07b2025 100644 --- a/flake.nix +++ b/flake.nix @@ -19,39 +19,10 @@ devShell = pkgs.mkShell { nativeBuildInputs = with pkgs; let - - renamed_grpc_tools = stdenv.mkDerivation { - name = "renamed-grpc-tools"; - version = "0.1"; - phases = [ "installPhase" ]; - installPhase = '' - mkdir -p $out/bin - cp -r ${protobuf}/include/ $out/bin/ - cp "${grpc-tools}/bin/protoc" $out/bin/grpc_tools_node_protoc - cp "${grpc-tools}/bin/grpc_node_plugin" $out/bin/grpc_tools_node_protoc_plugin - ''; - }; - - ts_protoc = stdenv.mkDerivation rec { - name = "protoc-gen-ts-wksp"; - src = fetchFromGitHub { - owner = "thesayyn"; - repo = "protoc-gen-ts"; - rev = "0.8.7"; - hash = "sha256-PGprtSPMRTodt/SD6gpEr/n22jiNqB1/C6HJGlDndLg="; - }; - buildInputs = [ git cacert nodejs bazel ]; - buildPhase = '' - export HOME=$(pwd) - mkdir -p $out/bin - npm ci - bazel build package - ''; - installPhase = '' - cp bazel-bin/package/package/protoc-gen-ts.js $out/bin/protoc-gen-ts - ''; - }; - + grpc-tools-node = + import ./nix-pkgs/grpc-tools-node.nix { inherit pkgs; }; + protoc-gen-ts = + import ./nix-pkgs/protoc-gen-ts.nix { inherit pkgs; }; in [ goreleaser go_1_19 @@ -75,8 +46,8 @@ libiconv bash-completion # local definition (see above) - renamed_grpc_tools - ts_protoc + grpc-tools-node + protoc-gen-ts ]; shellHook = '' diff --git a/nix-pkgs/grpc-tools-node.nix b/nix-pkgs/grpc-tools-node.nix new file mode 100644 index 0000000000..a33b7b47cc --- /dev/null +++ b/nix-pkgs/grpc-tools-node.nix @@ -0,0 +1,14 @@ +{ pkgs ? import { } }: +with pkgs; +stdenv.mkDerivation { + name = "grpc-tools-node"; + version = "0.1"; + phases = [ "installPhase" ]; + # For some reason, Node gRPC has its own 'protoc' binary + installPhase = '' + mkdir -p $out/bin + cp -r ${protobuf}/include/ $out/bin/ + cp "${grpc-tools}/bin/protoc" $out/bin/grpc_tools_node_protoc + cp "${grpc-tools}/bin/grpc_node_plugin" $out/bin/grpc_tools_node_protoc_plugin + ''; +} diff --git a/nix-pkgs/protoc-gen-ts.nix b/nix-pkgs/protoc-gen-ts.nix new file mode 100644 index 0000000000..5a14506179 --- /dev/null +++ b/nix-pkgs/protoc-gen-ts.nix @@ -0,0 +1,31 @@ +{ pkgs ? import { } }: +pkgs.buildBazelPackage rec { + bazel = pkgs.bazel_6; + + pname = "protoc-gen-ts"; + version = "0.8.7"; + + src = pkgs.fetchFromGitHub { + owner = "thesayyn"; + repo = pname; + rev = version; + hash = "sha256-PGprtSPMRTodt/SD6gpEr/n22jiNqB1/C6HJGlDndLg"; + }; + + fetchAttrs = { + sha256 = "sha256-VF9KyQl1UUPlJP4JoTIfPoJO3U+GGvtCQ4rj/2joGGs="; + }; + + nativeBuildInputs = [ pkgs.git pkgs.cacert pkgs.nodejs ]; + + bazelTargets = [ "//package:protoc-gen-ts" ]; + buildAttrs = { + preBuild = '' + mkdir -p $out/bin + ''; + installPhase = '' + cp bazel-bin/package/protoc-gen-ts.js $out/bin/protoc-gen-ts + ''; + }; + +} From 711f922ce0512d1b1b28613d18fd9d7bca0ce0f6 Mon Sep 17 00:00:00 2001 From: lostbean Date: Mon, 27 Nov 2023 19:10:59 -0300 Subject: [PATCH 2/3] try patching bazel fetched deps --- nix-pkgs/protoc-gen-ts.nix | 60 ++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/nix-pkgs/protoc-gen-ts.nix b/nix-pkgs/protoc-gen-ts.nix index 5a14506179..a528d49b41 100644 --- a/nix-pkgs/protoc-gen-ts.nix +++ b/nix-pkgs/protoc-gen-ts.nix @@ -1,5 +1,13 @@ { pkgs ? import { } }: -pkgs.buildBazelPackage rec { +let + ld_paths = pkgs.lib.makeLibraryPath [ + pkgs.stdenv.cc.cc + pkgs.stdenv.cc.libc + pkgs.glibc + ]; + ld_interpreter = + pkgs.lib.fileContents "${pkgs.stdenv.cc}/nix-support/dynamic-linker"; +in pkgs.buildBazelPackage rec { bazel = pkgs.bazel_6; pname = "protoc-gen-ts"; @@ -11,21 +19,49 @@ pkgs.buildBazelPackage rec { rev = version; hash = "sha256-PGprtSPMRTodt/SD6gpEr/n22jiNqB1/C6HJGlDndLg"; }; - + + LD_LIBRARY_PATH = ld_paths; + NIX_LD_LIBRARY_PATH = ld_paths; + NIX_LD = ld_interpreter; + LD = ld_interpreter; + fetchAttrs = { - sha256 = "sha256-VF9KyQl1UUPlJP4JoTIfPoJO3U+GGvtCQ4rj/2joGGs="; + sha256 = "sha256-gIeB/+GJiU9TMtMFB3RrmBwFSqEpSoM/HMMxT8A9AuM="; }; nativeBuildInputs = [ pkgs.git pkgs.cacert pkgs.nodejs ]; - bazelTargets = [ "//package:protoc-gen-ts" ]; - buildAttrs = { - preBuild = '' - mkdir -p $out/bin - ''; - installPhase = '' - cp bazel-bin/package/protoc-gen-ts.js $out/bin/protoc-gen-ts - ''; - }; + buildPhase = '' + bazel build --nobuild //package:protoc-gen-ts || true + + NODEBIN=$(bazel info output_base)/external/nodejs_linux_arm64/bin/nodejs/bin/node + echo ">>>>>>>>>>>>>>>>>>>>>" + INTER="${ld_interpreter}" + GCCLIB="${ld_paths}" + echo "$NODEBIN" + echo "$INTER" + echo "$GCCLIB" + echo ">>>>>>>>>>>>>>>>>>>>>" + echo $(sha256sum "$NODEBIN") + patchelf --set-rpath "$GCCLIB" "$NODEBIN" + echo $(sha256sum "$NODEBIN") + patchelf --set-interpreter "$INTER" "$NODEBIN" + echo $(sha256sum "$NODEBIN") + echo ">>>>>>>>>>>>>>>>>>>>>" + ldd -v "$NODEBIN" + file "$NODEBIN" + echo ">>>>>>>>>>>>>>>>>>>>>" + # bazel build //package:protoc-gen-ts --spawn_strategy=standalone --action_env=LD_LIBRARY_PATH=$LD_LIBRARY_PATH + bazel build package --action_env=LD_LIBRARY_PATH=$LD_LIBRARY_PATH --action_env=PATH=$PATH --action_env=NIX_LD=$LD --spawn_strategy=local + mkdir -p $out/bin + ''; + bazelTargets = [ "//package:protoc-gen-ts" ]; + installPhase = '' + ls -lah + ls -lah bazel-bin + ls -lah bazel-bin/package + cp bazel-bin/package/protoc-gen-ts.js $out/bin/protoc-gen-ts + ''; + buildAttrs = {}; } From 861f6fa435fd062088e051bdbb4105c1357af78f Mon Sep 17 00:00:00 2001 From: lostbean Date: Mon, 27 Nov 2023 19:55:06 -0300 Subject: [PATCH 3/3] Use fixed output on nix definition for protoc-gen-ts --- nix-pkgs/protoc-gen-ts.nix | 189 ++++++++++++++++++++++++++----------- 1 file changed, 135 insertions(+), 54 deletions(-) diff --git a/nix-pkgs/protoc-gen-ts.nix b/nix-pkgs/protoc-gen-ts.nix index a528d49b41..8801d821a8 100644 --- a/nix-pkgs/protoc-gen-ts.nix +++ b/nix-pkgs/protoc-gen-ts.nix @@ -1,67 +1,148 @@ { pkgs ? import { } }: +with pkgs; let - ld_paths = pkgs.lib.makeLibraryPath [ - pkgs.stdenv.cc.cc - pkgs.stdenv.cc.libc - pkgs.glibc - ]; - ld_interpreter = - pkgs.lib.fileContents "${pkgs.stdenv.cc}/nix-support/dynamic-linker"; -in pkgs.buildBazelPackage rec { - bazel = pkgs.bazel_6; + pkg_name = "protoc-gen-ts"; + pkg_version = "0.8.7"; - pname = "protoc-gen-ts"; - version = "0.8.7"; + # Using fixed output derivation so we can fetch things from outside + # and to avoid messing with Bazel's hermetic deps (mostly nodejs) + # we'll build it direclty using using Nix's nodejs. Very very hack solution + # after countless hours fighting Bazel deps vs nix sandbox. Maybe there a way easer + # way to tell bazel to use external nodejs but I couldn't figure it out. + # It seems the upstream repo is moving away from Bazel into Cargo. + deps = stdenv.mkDerivation rec { + name = "${pkg_name}-build"; - src = pkgs.fetchFromGitHub { - owner = "thesayyn"; - repo = pname; - rev = version; - hash = "sha256-PGprtSPMRTodt/SD6gpEr/n22jiNqB1/C6HJGlDndLg"; - }; + src = fetchFromGitHub { + owner = "thesayyn"; + repo = pkg_name; + rev = pkg_version; + hash = "sha256-PGprtSPMRTodt/SD6gpEr/n22jiNqB1/C6HJGlDndLg="; + }; - LD_LIBRARY_PATH = ld_paths; - NIX_LD_LIBRARY_PATH = ld_paths; - NIX_LD = ld_interpreter; - LD = ld_interpreter; + buildInputs = [ git cacert nodejs ]; - fetchAttrs = { - sha256 = "sha256-gIeB/+GJiU9TMtMFB3RrmBwFSqEpSoM/HMMxT8A9AuM="; - }; + # Setup TS/Node config files directly into the src repo + tsconfig_json = pkgs.writeText "tsconfig_json" '' + { + "compilerOptions": { + "target": "ES2020", + "module": "EsNext", + "moduleResolution": "node", + "outDir": "./.build", + "noImplicitAny": true + } + } + ''; - nativeBuildInputs = [ pkgs.git pkgs.cacert pkgs.nodejs ]; + package_json = pkgs.writeText "package_json" '' + { + "name": "example-node-nix", + "version": "1.0.0", + "main": "dist/index.js", + "bin": { + "example-node-nix": "dist/index.js" + }, + "scripts": { + "build": "tsc -p tsconfig.json && rollup -c rollup.config.js", + "start": "node dist/index.js" + }, + "dependencies": { + "@grpc/grpc-js": "^1.7.3", + "@types/google-protobuf": "^3.15.5", + "google-protobuf": "^3.19.1" + }, + "devDependencies": { + "@rollup/plugin-commonjs": "^23.0.2", + "@rollup/plugin-node-resolve": "^15.0.1", + "@rollup/plugin-typescript": "^11.1.5", + "@tsconfig/node16": "^1.0.1", + "@tsconfig/node16-strictest": "^1.0.0", + "@types/node": "^18.7.14", + "rollup": "<3.0.0", + "typescript": "^4.8.2", + "tslib": "^2.6.2" + } + } + ''; - buildPhase = '' - bazel build --nobuild //package:protoc-gen-ts || true + rollup_config_js = pkgs.writeText "rollup_config_js" '' + import nodeResolve from '@rollup/plugin-node-resolve'; + import commonjs from '@rollup/plugin-commonjs'; + import typescript from '@rollup/plugin-typescript'; + import fs from "node:fs"; - NODEBIN=$(bazel info output_base)/external/nodejs_linux_arm64/bin/nodejs/bin/node - echo ">>>>>>>>>>>>>>>>>>>>>" - INTER="${ld_interpreter}" - GCCLIB="${ld_paths}" - echo "$NODEBIN" - echo "$INTER" - echo "$GCCLIB" - echo ">>>>>>>>>>>>>>>>>>>>>" - echo $(sha256sum "$NODEBIN") - patchelf --set-rpath "$GCCLIB" "$NODEBIN" - echo $(sha256sum "$NODEBIN") - patchelf --set-interpreter "$INTER" "$NODEBIN" - echo $(sha256sum "$NODEBIN") - echo ">>>>>>>>>>>>>>>>>>>>>" - ldd -v "$NODEBIN" - file "$NODEBIN" - echo ">>>>>>>>>>>>>>>>>>>>>" - # bazel build //package:protoc-gen-ts --spawn_strategy=standalone --action_env=LD_LIBRARY_PATH=$LD_LIBRARY_PATH - bazel build package --action_env=LD_LIBRARY_PATH=$LD_LIBRARY_PATH --action_env=PATH=$PATH --action_env=NIX_LD=$LD --spawn_strategy=local + const executable = () => { + return { + name: 'executable', + writeBundle: (options) => { + fs.chmodSync(options.file, '755'); + }, + }; + }; - mkdir -p $out/bin - ''; - bazelTargets = [ "//package:protoc-gen-ts" ]; + export default { + input: "./index.ts", + output: { + file: '.bin/protoc-gen-ts.js', + format: 'cjs' + }, + plugins: [ + typescript(), + nodeResolve(), + commonjs(), + executable() + ], + onwarn(message, warn) { + if (message.code === "EVAL" || message.code == "THIS_IS_UNDEFINED") { + return; + } + warn(message); + } + } + ''; + + buildPhase = '' + cp ${package_json} src/package.json + cp ${rollup_config_js} src/rollup.config.js + cp ${tsconfig_json} src/tsconfig.json + cd src + + echo ">> Create .tmp folder as Nix sandbox are homeless and nodejs needs one" + mkdir -p .tmp + export HOME=$(pwd)/.tmp + + echo ">> Intsalling TS/JS/nodejs deps" + npm install + + echo ">> Patching npm's TS and RollUp scripts to work on Nix" + patchShebangs --build -- node_modules/typescript/bin/tsc + patchShebangs --build -- node_modules/rollup/dist/bin/rollup + + echo ">> Building" + npm run build + ''; + + installPhase = '' + mkdir -p $out/bin + cp .bin/protoc-gen-ts.js $out/bin/protoc-gen-ts + ''; + + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = "sha256-cIIFpEXlg/UogK4QskIeWFDuXReFDBKokddTB6juTZo="; + }; + +in stdenv.mkDerivation rec { + name = pkg_name; + version = pkg_version; + phases = [ "installPhase" ]; + + # Fixed Output derivation can't have Nix references (paths /nix/...) on it + # and we need to add it on this second (and normal) derivation. We're adding + # a shebang to call nodejs in the compiled script. installPhase = '' - ls -lah - ls -lah bazel-bin - ls -lah bazel-bin/package - cp bazel-bin/package/protoc-gen-ts.js $out/bin/protoc-gen-ts + mkdir -p $out/bin + echo "$(echo '#!${nodejs}/bin/node' && cat '${deps}/bin/protoc-gen-ts')" > $out/bin/protoc-gen-ts ''; - buildAttrs = {}; }