Skip to content

Commit

Permalink
Merge pull request NixOS#235466 from 06kellyjac/semgrep
Browse files Browse the repository at this point in the history
  • Loading branch information
@Janik-Haag
Janik-Haag authored Jun 26, 2023
2 parents ea12f27 + f3dde5b commit 7f8a16b
Show file tree
Hide file tree
Showing 4 changed files with 102 additions and 69 deletions.
55 changes: 19 additions & 36 deletions pkgs/tools/security/semgrep/common.nix
View file
Original file line number Diff line number Diff line change
@@ -1,54 +1,39 @@
{ lib, fetchFromGitHub, fetchzip, stdenv }:
{ lib }:

rec {
version = "1.15.0";
version = "1.27.0";

src = fetchFromGitHub {
owner = "returntocorp";
repo = "semgrep";
rev = "v${version}";
sha256 = "sha256-x+AOt6nn2hN4MODFZCvlq0kZ3VLoS7rVcFGGCEssIu0=";
};
srcHash = "sha256-F6n3LQY4a5sO6c8SMQF9YjjgOS+v2SH+UQPwhg2EX7Q=";

# submodule dependencies
# these are fetched so we:
# 1. don't fetch the many submodules we don't need
# 2. avoid fetchSubmodules since it's prone to impurities
submodules = {
"cli/src/semgrep/lang" = fetchFromGitHub {
owner = "returntocorp";
repo = "semgrep-langs";
rev = "08656cdefc9e6818c64e168cf51ee1e76ea8829e";
sha256 = "sha256-vYf33JhfvEDmt/VW0hBOmqailIERS0GdUgrPuCxWt9I=";
};
"cli/src/semgrep/semgrep_interfaces" = fetchFromGitHub {
"cli/src/semgrep/semgrep_interfaces" = {
owner = "returntocorp";
repo = "semgrep-interfaces";
rev = "ba9241ca8f13dea72a4ca5c5eae99f45c071c8b4";
sha256 = "sha256-2rcMmN42445AivcyYLPeE+HBYOyxJijQME1UUr9HISA=";
rev = "213f67abea73546ca6111e1bbf0ef96aa917c940";
hash = "sha256-HeNHJkTje9j16+dwsfyMhoqQn/J18q/7XvQPRwgTw/Y=";
};
};

# fetch pre-built semgrep-core since the ocaml build is complex and relies on
# the opam package manager at some point
core = rec {
data = {
x86_64-linux = {
suffix = "-ubuntu-16.04.tgz";
sha256 = "sha256-vLtV1WAnOD6HhgrWYIP0NfXHKfvXORksdNp5UTG1QWc=";
};
x86_64-darwin = {
suffix = "-osx.zip";
sha256 = "sha256-6+ENjOOIJ5TSjpnJ5pDudblrWj/FLUe66UGr6V9c0HQ=";
};
# pulling it out of the python wheel as r2c no longer release a built binary
# on github releases
core = {
x86_64-linux = {
platform = "any";
hash = "sha256-cRj81dXpAE6S0EXajsRikOIAPzlUf42FhiDCWjv+wZQ=";
};
x86_64-darwin = {
platform = "macosx_10_14_x86_64";
hash = "sha256-jqfGVZGF/DFgXkr7kQg6QyqEELSr8AKE3Ga8kTftnIY=";
};
src = let
inherit (stdenv.hostPlatform) system;
selectSystemData = data: data.${system} or (throw "Unsupported system: ${system}");
inherit (selectSystemData data) suffix sha256;
in fetchzip {
url = "https://github.com/returntocorp/semgrep/releases/download/v${version}/semgrep-v${version}${suffix}";
inherit sha256;
aarch64-darwin = {
platform = "macosx_11_0_arm64";
hash = "sha256-e/uCSRMdbVD0lvc0hukbiUzheqRNIIh1LgMq6Ae7JYI=";
};
};

Expand All @@ -66,7 +51,5 @@ rec {
'';
license = licenses.lgpl21Plus;
maintainers = with maintainers; [ jk ambroisie ];
# limited by semgrep-core
platforms = [ "x86_64-linux" "x86_64-darwin" ];
};
}
17 changes: 13 additions & 4 deletions pkgs/tools/security/semgrep/default.nix
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
{ lib
, fetchFromGitHub
, callPackage
, semgrep-core
, buildPythonApplication
, pythonPackages
Expand All @@ -11,12 +10,20 @@
}:

let
common = callPackage ./common.nix { };
common = import ./common.nix { inherit lib; };
in
buildPythonApplication rec {
pname = "semgrep";
inherit (common) src version;
inherit (common) version;
src = fetchFromGitHub {
owner = "returntocorp";
repo = "semgrep";
rev = "v${version}";
hash = common.srcHash;
};

# prepare a subset of the submodules as we only need a handful
# and there are many many submodules total
postPatch = (lib.concatStringsSep "\n" (lib.mapAttrsToList
(
path: submodule: ''
Expand All @@ -27,7 +34,7 @@ buildPythonApplication rec {
ln -s ${submodule}/ ${path}
''
)
common.submodules)) + ''
passthru.submodulesSubset)) + ''
cd cli
'';

Expand Down Expand Up @@ -97,10 +104,12 @@ buildPythonApplication rec {

passthru = {
inherit common;
submodulesSubset = lib.mapAttrs (k: args: fetchFromGitHub args) common.submodules;
updateScript = ./update.sh;
};

meta = common.meta // {
description = common.meta.description + " - cli";
inherit (semgrep-core.meta) platforms;
};
}
39 changes: 35 additions & 4 deletions pkgs/tools/security/semgrep/semgrep-core.nix
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,52 @@
{ lib, stdenvNoCC, callPackage }:
{ lib, stdenvNoCC, fetchPypi, unzip }:

let
common = callPackage ./common.nix { };
common = import ./common.nix { inherit lib; };
in
stdenvNoCC.mkDerivation rec {
pname = "semgrep-core";
inherit (common) version;
inherit (common.core) src;
# fetch pre-built semgrep-core since the ocaml build is complex and relies on
# the opam package manager at some point
# pulling it out of the python wheel as r2c no longer release a built binary
# on github releases
src =
let
inherit (stdenvNoCC.hostPlatform) system;
data = common.core.${system} or (throw "Unsupported system: ${system}");
in
fetchPypi rec {
pname = "semgrep";
inherit version;
format = "wheel";
dist = python;
python = "cp37.cp38.cp39.py37.py38.py39";
inherit (data) platform hash;
};

nativeBuildInputs = [ unzip ];

# _tryUnzip from unzip's setup-hook doesn't recognise .whl
# "do not know how to unpack source archive"
# perform unpack by hand
unpackPhase = ''
runHook preUnpack
LANG=en_US.UTF-8 unzip -qq "$src"
runHook postUnpack
'';

dontConfigure = true;
dontBuild = true;

installPhase = ''
runHook preInstall
install -Dm 755 -t $out/bin semgrep-core
install -Dm 755 -t $out/bin semgrep-${version}.data/purelib/semgrep/bin/semgrep-core
runHook postInstall
'';

meta = common.meta // {
description = common.meta.description + " - core binary";
sourceProvenance = with lib.sourceTypes; [ binaryNativeCode ];
platforms = lib.attrNames common.core;
};
}
60 changes: 35 additions & 25 deletions pkgs/tools/security/semgrep/update.sh
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/usr/bin/env nix-shell
#!nix-shell -i bash -p curl gnused jq
#!nix-shell -i bash -p curl gnused jq nix-prefetch

set -euxo pipefail

Expand Down Expand Up @@ -33,7 +33,7 @@ NEW_VERSION=$(
)
# trim v prefix
NEW_VERSION="${NEW_VERSION:1}"
OLD_VERSION="$(instantiateClean semgrep.common.version)"
OLD_VERSION="$(instantiateClean semgrep.passthru.common.version)"

if [[ "$OLD_VERSION" == "$NEW_VERSION" ]]; then
echo "Already up to date"
Expand All @@ -50,43 +50,54 @@ fetchgithub() {
set -eo pipefail
}

fetchzip() {
set +eo pipefail
nix-build -E "with import $NIXPKGS_ROOT {}; fetchzip {url = \"$1\"; sha256 = lib.fakeSha256; }" 2>&1 >/dev/null | grep "got:" | cut -d':' -f2 | sed 's| ||g'
set -eo pipefail
fetch_arch() {
VERSION=$1
PLATFORM=$2
nix-prefetch "{ fetchPypi }:
fetchPypi rec {
pname = \"semgrep\";
version = \"$VERSION\";
format = \"wheel\";
dist = python;
python = \"cp37.cp38.cp39.py37.py38.py39\";
platform = \"$PLATFORM\";
}
"
}

replace "$OLD_VERSION" "$NEW_VERSION" "$COMMON_FILE"

echo "Updating src"

OLD_HASH="$(instantiateClean semgrep.common.src.outputHash)"
OLD_HASH="$(instantiateClean semgrep.passthru.common.srcHash)"
echo "Old hash $OLD_HASH"
TMP_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
replace "$OLD_HASH" "$TMP_HASH" "$COMMON_FILE"
NEW_HASH="$(fetchgithub semgrep.common.src)"
NEW_HASH="$(fetchgithub semgrep.src)"
echo "New hash $NEW_HASH"
replace "$TMP_HASH" "$NEW_HASH" "$COMMON_FILE"

echo "Updated src"

# loop through platforms for core
nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.common.core.data" --eval --strict --json \
| jq '.[]' -r \
| while read -r PLATFORM; do
echo "Updating core for $PLATFORM"
SUFFIX=$(instantiateClean semgrep.common.core.data."$PLATFORM".suffix)
OLD_HASH=$(instantiateClean semgrep.common.core.data."$PLATFORM".sha256)
echo "Old hash $OLD_HASH"

NEW_URL="https://github.com/returntocorp/semgrep/releases/download/v$NEW_VERSION/semgrep-v$NEW_VERSION$SUFFIX"
NEW_HASH="$(fetchzip "$NEW_URL")"
echo "New hash $NEW_HASH"
update_core_platform() {
SYSTEM=$1
echo "Updating core src $SYSTEM"

PLATFORM="$(instantiateClean "semgrep.passthru.common.core.$SYSTEM.platform")"

OLD_HASH="$(instantiateClean "semgrep.passthru.common.core.$SYSTEM.hash")"
echo "Old core hash $OLD_HASH"
NEW_HASH="$(fetch_arch "$NEW_VERSION" "$PLATFORM")"
echo "New core hash $NEW_HASH"
replace "$OLD_HASH" "$NEW_HASH" "$COMMON_FILE"

echo "Updated core for $PLATFORM"
done
echo "Updated core src $SYSTEM"
}

update_core_platform "x86_64-linux"
update_core_platform "x86_64-darwin"
update_core_platform "aarch64-darwin"

OLD_PWD=$PWD
TMPDIR="$(mktemp -d)"
Expand All @@ -109,7 +120,7 @@ nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.pas
echo "Updating $SUBMODULE"
OLD_REV=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".rev)
echo "Old commit $OLD_REV"
OLD_HASH=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".outputHash)
OLD_HASH=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".hash)
echo "Old hash $OLD_HASH"

NEW_REV=$(get_submodule_commit "$SUBMODULE")
Expand All @@ -120,13 +131,12 @@ nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.pas
continue
fi

NEW_URL=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".url | sed "s@$OLD_REV@$NEW_REV@g")
NEW_HASH=$(nix --experimental-features nix-command hash to-sri "sha256:$(nix-prefetch-url "$NEW_URL")")
NEW_URL=$(instantiateClean semgrep.passthru.submodulesSubset."$SUBMODULE".url | sed "s@$OLD_REV@$NEW_REV@g")

TMP_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
replace "$OLD_REV" "$NEW_REV" "$COMMON_FILE"
replace "$OLD_HASH" "$TMP_HASH" "$COMMON_FILE"
NEW_HASH="$(fetchgithub semgrep.passthru.common.submodules."$SUBMODULE")"
NEW_HASH="$(fetchgithub semgrep.passthru.submodulesSubset."$SUBMODULE")"
echo "New hash $NEW_HASH"
replace "$TMP_HASH" "$NEW_HASH" "$COMMON_FILE"

Expand Down

0 comments on commit 7f8a16b

Please sign in to comment.