diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 8a593df..b6527f9 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -36,6 +36,8 @@ export const tulipApi = createApi({ ...query, includeTags: query.includeTags.length > 0 ? query.includeTags : undefined, excludeTags: query.excludeTags.length > 0 ? query.excludeTags : undefined, + includeFuzzyHashes: query.includeFuzzyHashes.length > 0 ? query.includeFuzzyHashes : undefined, + excludeFuzzyHashes: query.excludeFuzzyHashes.length > 0 ? query.excludeFuzzyHashes : undefined, }), }), }), diff --git a/frontend/src/components/Corrie.tsx b/frontend/src/components/Corrie.tsx index e943a26..e5a15df 100644 --- a/frontend/src/components/Corrie.tsx +++ b/frontend/src/components/Corrie.tsx @@ -7,7 +7,7 @@ import { START_FILTER_KEY, END_FILTER_KEY, CORRELATION_MODE_KEY, - FLOW_LIST_REFETCH_INTERVAL_MS, + FLOW_LIST_REFETCH_INTERVAL_MS, SIMILARITY_FILTER_KEY, } from "../const"; import useDebounce from "../hooks/useDebounce"; @@ -29,8 +29,8 @@ export const Corrie = () => { const includeTags = useAppSelector((state) => state.filter.includeTags); const excludeTags = useAppSelector((state) => state.filter.excludeTags); const filterTags = useAppSelector((state) => state.filter.filterTags); - const filterFlags = useAppSelector((state) => state.filter.filterFlags); - const filterFlagids = useAppSelector((state) => state.filter.filterFlagids); + const includeFuzzyHashes = useAppSelector((state) => state.filter.includeFuzzyHashes); + const excludeFuzzyHashes = useAppSelector((state) => state.filter.excludeFuzzyHashes); const [searchParams, setSearchParams] = useSearchParams(); @@ -38,6 +38,7 @@ export const Corrie = () => { const service = services && services.find((s) => s.name == service_name); const text_filter = searchParams.get(TEXT_FILTER_KEY) ?? undefined; + const similarity = searchParams.get(SIMILARITY_FILTER_KEY) ?? undefined; const from_filter = searchParams.get(START_FILTER_KEY) ?? undefined; const to_filter = searchParams.get(END_FILTER_KEY) ?? undefined; @@ -54,8 +55,9 @@ export const Corrie = () => { includeTags: includeTags, excludeTags: excludeTags, tags: filterTags, - flags: filterFlags, - flagids: filterFlagids, + similarity: similarity, + includeFuzzyHashes: includeFuzzyHashes, + excludeFuzzyHashes: excludeFuzzyHashes }, { refetchOnMountOrArgChange: true, diff --git a/frontend/src/components/FlowList.tsx b/frontend/src/components/FlowList.tsx index b20272a..6351181 100644 --- a/frontend/src/components/FlowList.tsx +++ b/frontend/src/components/FlowList.tsx @@ -12,10 +12,10 @@ import { TEXT_FILTER_KEY, START_FILTER_KEY, END_FILTER_KEY, - FLOW_LIST_REFETCH_INTERVAL_MS, + FLOW_LIST_REFETCH_INTERVAL_MS, SIMILARITY_FILTER_KEY, } from "../const"; import { useAppSelector, useAppDispatch } from "../store"; -import { toggleFilterTag } from "../store/filter"; +import {toggleFilterFuzzyHashes, toggleFilterTag} from "../store/filter"; import { HeartIcon, FilterIcon, LinkIcon } from "@heroicons/react/solid"; import { HeartIcon as EmptyHeartIcon } from "@heroicons/react/outline"; @@ -44,10 +44,12 @@ export function FlowList() { const { data: services } = useGetServicesQuery(); const filterTags = useAppSelector((state) => state.filter.filterTags); - const filterFlags = useAppSelector((state) => state.filter.filterFlags); - const filterFlagids = useAppSelector((state) => state.filter.filterFlagids); const includeTags = useAppSelector((state) => state.filter.includeTags); const excludeTags = useAppSelector((state) => state.filter.excludeTags); + const includeFuzzyHashes = useAppSelector((state) => state.filter.includeFuzzyHashes); + const excludeFuzzyHashes = useAppSelector((state) => state.filter.excludeFuzzyHashes); + const fuzzyHashes = useAppSelector((state) => state.filter.fuzzyHashes); + const fuzzyHashIds = useAppSelector((state) => state.filter.fuzzyHashIds); const dispatch = useAppDispatch(); @@ -61,6 +63,7 @@ export function FlowList() { const service = services?.find((s) => s.name == service_name); const text_filter = searchParams.get(TEXT_FILTER_KEY) ?? undefined; + const similarity = searchParams.get(SIMILARITY_FILTER_KEY) ?? undefined; const from_filter = searchParams.get(START_FILTER_KEY) ?? undefined; const to_filter = searchParams.get(END_FILTER_KEY) ?? undefined; @@ -75,10 +78,11 @@ export function FlowList() { to_time: to_filter, service: "", // FIXME tags: filterTags, - flags: filterFlags, - flagids: filterFlagids, includeTags: includeTags, - excludeTags: excludeTags + excludeTags: excludeTags, + similarity: similarity, + includeFuzzyHashes: includeFuzzyHashes, + excludeFuzzyHashes: excludeFuzzyHashes }, { refetchOnMountOrArgChange: true, @@ -197,6 +201,24 @@ export function FlowList() { )} + {showFilters && ( +
+

+ Similarity filter +

+
+ {(fuzzyHashes ?? []).map((fuzzyHash, i) => ( + dispatch(toggleFilterFuzzyHashes([fuzzyHash, fuzzyHashIds[i]]))} + > + ))} +
+
+ )}
setFlowIndex(index)} key={flow._id.$oid} - className="focus-visible:rounded-md" - //style={{ paddingTop: '1em' }} - > + className={classNames({ + "flex-1": true, + [classes.list_container]: true, + "sidebar-loading": isLoading, + })} + > {flow.duration}ms ); - return ( + + + const DEFAULT = [243, 244, 246] + const GREEN = [134, 239, 172] + const RED = [252, 165, 165] + + var color: number[] + if (flow.similarity != undefined) + color = GREEN.map((g, i) => ((g*flow.similarity) + (RED[i]*(1-flow.similarity)))); + else + color = DEFAULT; + + return (
  • { + setValue(event.target.value); + searchParams.set(SIMILARITY_FILTER_KEY, event.target.value) + setSearchParams(searchParams); + }; + + return ( +
    +
    + + Similarity: + + +
    + +
    + ); +} + export function Header() { let [searchParams] = useSearchParams(); const { setToLastnTicks, currentTick, setTimeParam } = useMessyTimeStuff(); @@ -330,17 +369,17 @@ export function Header() { setTimeParam("", END_FILTER_KEY); }); - return ( - <> - -
    🌷
    - -
    - -
    -
    - - + return ( + <> + +
    🌷
    + +
    + +
    +
    + +
    @@ -362,6 +401,7 @@ export function Header() { Graph view
    +
    diff --git a/frontend/src/const.ts b/frontend/src/const.ts index b2bea03..5251fea 100644 --- a/frontend/src/const.ts +++ b/frontend/src/const.ts @@ -2,6 +2,7 @@ export const API_BASE_PATH = "/api"; export const TEXT_FILTER_KEY = "text"; export const SERVICE_FILTER_KEY = "service"; +export const SIMILARITY_FILTER_KEY = "similarity"; export const START_FILTER_KEY = "start"; export const END_FILTER_KEY = "end"; export const FIRST_DIFF_KEY = "first"; diff --git a/frontend/src/pages/FlowView.tsx b/frontend/src/pages/FlowView.tsx index 1b5e42d..aa0b659 100644 --- a/frontend/src/pages/FlowView.tsx +++ b/frontend/src/pages/FlowView.tsx @@ -27,6 +27,8 @@ import { useToSinglePythonRequestQuery, useGetFlagRegexQuery, } from "../api"; +import {toggleFilterFuzzyHashes, toggleFilterTag} from "../store/filter"; +import {useAppDispatch, useAppSelector} from "../store"; import escapeStringRegexp from 'escape-string-regexp'; const SECONDARY_NAVBAR_HEIGHT = 50; @@ -87,7 +89,7 @@ function highlightText(flowText: string, search_string: string, flag_string: str } const searchClasses = "bg-orange-200 rounded-sm" const flagClasses = "bg-red-200 rounded-sm" - return { parts.map((part, i) => + return { parts.map((part, i) => { part } ) @@ -301,89 +303,110 @@ function formatIP(ip: string) { function FlowOverview({ flow }: { flow: FullFlow }) { const FILTER_KEY = TEXT_FILTER_KEY; let [searchParams, setSearchParams] = useSearchParams(); + const dispatch = useAppDispatch(); return ( -
    - {flow.signatures?.length > 0 ? ( -
    -
    Suricata
    +
    + {flow.signatures?.length > 0 ? ( +
    +
    Suricata
    +
    + {flow.signatures.map((sig) => { + return ( +
    +
    +
    Message:
    +
    {sig.msg}
    +
    +
    +
    Rule ID:
    +
    {sig.id}
    +
    +
    +
    Action taken:
    +
    + {sig.action} +
    +
    +
    + ); + })} +
    +
    + ) : undefined} +
    +
    Meta
    - {flow.signatures.map((sig) => { - return ( -
    -
    -
    Message:
    -
    {sig.msg}
    -
    -
    -
    Rule ID:
    -
    {sig.id}
    -
    -
    -
    Action taken:
    -
    - {sig.action} -
    -
    -
    - ); - })} -
    -
    - ) : undefined} -
    -
    Meta
    -
    -
    Source:
    - -
    -
    Tags:
    -
    [{flow.tags.join(", ")}]
    -
    Flags:
    -
    - [{flow.flags.map((query, i) => ( - +
    Source:
    + +
    +
    Tags:
    +
    + [{flow.tags.map((tag, i) => ( + + {i > 0 ? ', ' : ''} + dispatch(toggleFilterTag(tag))}> + {tag} + + + ))}] +
    +
    +
    Flags:
    +
    + [{flow.flags.map((query, i) => ( + {i > 0 ? ', ' : ''} - + ))}] -
    -
    Flagids:
    -
    - [{flow.flagids.map((query, i) => ( - +
    +
    +
    Flagids:
    +
    + [{flow.flagids.map((query, i) => ( + {i > 0 ? ', ' : ''} - + ))}]
    +
    Nilsimsa hash:
    + +
    Source - Target (Duration):
    @@ -420,7 +443,7 @@ export function FlowView() { async function copyAsPwn() { if (flow?._id.$oid) { - const { data } = await triggerPwnToolsQuery(flow?._id.$oid); + const {data} = await triggerPwnToolsQuery(flow?._id.$oid); console.log(data); return data || ""; } @@ -505,7 +528,7 @@ export function FlowView() { onMouseDown={(e) => { if( e.button === 1 ) { // handle opening in new tab window.open(`/flow/${flow.parent_id.$oid}?${searchParams}`, '_blank') - } else if (e.button === 0) { + } else if (e.button === 0) { navigate(`/flow/${flow.parent_id.$oid}?${searchParams}`) } }} diff --git a/frontend/src/store/filter.ts b/frontend/src/store/filter.ts index 75770f8..852595d 100644 --- a/frontend/src/store/filter.ts +++ b/frontend/src/store/filter.ts @@ -1,11 +1,15 @@ import { createSlice, PayloadAction } from "@reduxjs/toolkit"; +// Note: all off these states are immutable and can only be changed through overwrite export interface TulipFilterState { filterTags: string[]; - filterFlags: string[]; - filterFlagids: string[]; includeTags: string[]; excludeTags: string[]; + // can't use Map because immutable bs + fuzzyHashes: string[]; + fuzzyHashIds: string[]; + includeFuzzyHashes: string[]; + excludeFuzzyHashes: string[]; // startTick?: number; // endTick?: number; // service?: string; @@ -16,8 +20,10 @@ const initialState: TulipFilterState = { includeTags: [], excludeTags: [], filterTags: [], - filterFlags: [], - filterFlagids: [], + fuzzyHashes: [], + fuzzyHashIds: [], + includeFuzzyHashes: [], + excludeFuzzyHashes: [], }; export const filterSlice = createSlice({ @@ -54,19 +60,42 @@ export const filterSlice = createSlice({ } } }, - toggleFilterFlags: (state, action: PayloadAction) => { - state.filterFlags = state.filterFlags.includes(action.payload) - ? state.filterFlags.filter((t) => t !== action.payload) - : [...state.filterFlags, action.payload]; - }, - toggleFilterFlagids: (state, action: PayloadAction) => { - state.filterFlagids = state.filterFlagids.includes(action.payload) - ? state.filterFlagids.filter((t) => t !== action.payload) - : [...state.filterFlagids, action.payload]; - }, + toggleFilterFuzzyHashes: (state, action: PayloadAction) => { + var fuzzyHashes = action.payload[0] + var id = action.payload[1] + var included = state.includeFuzzyHashes.includes(fuzzyHashes) + var excluded = state.excludeFuzzyHashes.includes(fuzzyHashes) + + // If the fuzzyHashes hash is new cache it + if(!state.fuzzyHashes.includes(fuzzyHashes)) { + state.fuzzyHashes = [...state.fuzzyHashes, fuzzyHashes] + state.fuzzyHashIds = [...state.fuzzyHashIds, id] + } + + // If a user clicks a 'included' fuzzyHashes hash, the hash should be 'excluded' instead. + if (included) { + // Remove from included + state.includeFuzzyHashes = state.includeFuzzyHashes.filter((t) => t !== fuzzyHashes); + + // Add to excluded + state.excludeFuzzyHashes = [...state.excludeFuzzyHashes, fuzzyHashes] + } else { + // If the user clicks on an 'excluded' fuzzyHashes hash, the hash should be 'unset' from both include / exclude tags + if (excluded) { + // Remove from excluded + state.excludeFuzzyHashes = state.excludeFuzzyHashes.filter((t) => t !== fuzzyHashes); + } else { + if (!included && !excluded) { + // The tag was disabled, so it should be added to included now + state.includeFuzzyHashes = [...state.includeFuzzyHashes, fuzzyHashes] + } + } + } + } }, }); -export const { toggleFilterTag } = filterSlice.actions; + +export const { toggleFilterTag, toggleFilterFuzzyHashes } = filterSlice.actions; export default filterSlice.reducer; diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 5af9d5b..8a3413b 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -16,6 +16,8 @@ export interface Flow { flagids: string[]; suricata: number[]; filename: string; + fuzzy_hash: string; + similarity: number } export interface TickInfo { @@ -59,8 +61,9 @@ export interface FlowsQuery { includeTags: string[]; excludeTags: string[]; tags: string[]; - flags: string[]; - flagids: string[]; + similarity?: string; + includeFuzzyHashes: string[]; + excludeFuzzyHashes: string[]; } export type Service = { diff --git a/services/api/Dockerfile-api b/services/api/Dockerfile-api index 5ef5300..a556b17 100644 --- a/services/api/Dockerfile-api +++ b/services/api/Dockerfile-api @@ -4,7 +4,7 @@ COPY ./requirements.txt /app/requirements.txt WORKDIR /app -RUN pip install -r requirements.txt +RUN BUILD_LIB=1 pip install -r requirements.txt COPY . /app diff --git a/services/api/requirements.txt b/services/api/requirements.txt index 3f28156..a059968 100644 --- a/services/api/requirements.txt +++ b/services/api/requirements.txt @@ -2,3 +2,4 @@ Flask_Cors pymongo Flask requests +nilsimsa \ No newline at end of file diff --git a/services/api/webservice.py b/services/api/webservice.py index d4246c4..b9974b1 100755 --- a/services/api/webservice.py +++ b/services/api/webservice.py @@ -23,6 +23,8 @@ # along with Flower. If not, see . import traceback + +import nilsimsa from flask import Flask, Response, send_file from configurations import services, traffic_dir, start_date, tick_length, flag_regex @@ -62,8 +64,25 @@ def getTickInfo(): @application.route('/query', methods=['POST']) def query(): - json = request.get_json() - result = db.getFlowList(json) + filter = request.get_json() + result = db.getFlowList(filter) + similarity = int(filter["similarity"]) if "similarity" in filter else 90 + + # adapt similarity to nilsimsa compare value + similarity = (similarity*256/100) - 128 + + if "includeFuzzyHashes" in filter: + result = [x for x in result if all(nilsimsa.compare_digests(hash, x["fuzzy_hash"]) >= similarity for hash in filter["includeFuzzyHashes"])] + for x in result: + x['similarity'] = ((sum(nilsimsa.compare_digests(hash, x["fuzzy_hash"]) for hash in filter["includeFuzzyHashes"]) / len(filter["includeFuzzyHashes"]))+128)/256 + + if "excludeFuzzyHashes" in filter: + result = [x for x in result if all(nilsimsa.compare_digests(hash, x["fuzzy_hash"]) < similarity for hash in filter["excludeFuzzyHashes"])] + for x in result: + similarity = ((-sum(nilsimsa.compare_digests(hash, x["fuzzy_hash"]) for hash in filter["excludeFuzzyHashes"]) / len(filter["excludeFuzzyHashes"]))+128)/256 + x['similarity'] = (similarity + x['similarity'])/2 if 'similarity' in x else similarity + + return return_json_response(result) @application.route('/tags') diff --git a/services/go-importer/cmd/assembler/main.go b/services/go-importer/cmd/assembler/main.go index 8962d8c..b81bba2 100644 --- a/services/go-importer/cmd/assembler/main.go +++ b/services/go-importer/cmd/assembler/main.go @@ -1,6 +1,7 @@ package main import ( + "github.com/chikamim/nilsimsa" "go-importer/internal/pkg/db" "flag" @@ -75,10 +76,19 @@ func reassemblyCallback(entry db.FlowEntry) { // Parsing HTTP will decode encodings to a plaintext format ParseHttpFlow(&entry) + allData := make([]byte, 0) + for idx := 0; idx < len(entry.Flow); idx++ { + flowItem := entry.Flow[idx] + allData = append(allData[:], []byte(flowItem.Data)[:]...) + } + + fuzzyHash := nilsimsa.HexSum(allData) + // Apply flag in / flagout if *flag_regex != "" { ApplyFlagTags(&entry, flag_regex) } + entry.Fuzzy_hash = fuzzyHash //Apply flagid in / out if *flagid { diff --git a/services/go-importer/go.mod b/services/go-importer/go.mod index 38d358b..3d593af 100644 --- a/services/go-importer/go.mod +++ b/services/go-importer/go.mod @@ -4,9 +4,10 @@ go 1.13 require ( github.com/andybalholm/brotli v1.0.4 + github.com/chikamim/nilsimsa v0.0.0-20200618014539-84540bbf06dd // indirect + github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184 github.com/fsnotify/fsnotify v1.5.4 github.com/google/gopacket v1.1.19 github.com/tidwall/gjson v1.14.1 go.mongodb.org/mongo-driver v1.9.1 - github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184 ) diff --git a/services/go-importer/go.sum b/services/go-importer/go.sum index dd6a23f..0d62ad0 100644 --- a/services/go-importer/go.sum +++ b/services/go-importer/go.sum @@ -1,7 +1,10 @@ github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/chikamim/nilsimsa v0.0.0-20200618014539-84540bbf06dd h1:7NiFDBnypHw4sI70R6+jpaExOEj9bA8wDY2GqTQcwvo= +github.com/chikamim/nilsimsa v0.0.0-20200618014539-84540bbf06dd/go.mod h1:Cc/DVLWhhAlZSuVPhfGw/2mUYLAPtGGNG0d6ha1H2pc= github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184 h1:8yL+85JpbwrIc6m+7N1iYrjn/22z68jwrTIBOJHNe4k= github.com/cloudflare/ahocorasick v0.0.0-20210425175752-730270c3e184/go.mod h1:tGWUZLZp9ajsxUOnHmFFLnqnlKXsCn6GReG4jAD59H0= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -19,17 +22,29 @@ github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo= github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -78,5 +93,7 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/go-importer/internal/pkg/db/db.go b/services/go-importer/internal/pkg/db/db.go index 9aac3d5..641a972 100644 --- a/services/go-importer/internal/pkg/db/db.go +++ b/services/go-importer/internal/pkg/db/db.go @@ -46,6 +46,7 @@ type FlowEntry struct { Size int Flags []string Flagids []string + Fuzzy_hash string } type Database struct {