diff --git a/src/utils.go b/src/utils.go deleted file mode 100644 index 98e4344f..00000000 --- a/src/utils.go +++ /dev/null @@ -1,696 +0,0 @@ -package main - -import ( - "crypto/sha256" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "log" - "math/rand" - "mime" - "net/http" - "net/url" - "os" - "regexp" - "strconv" - "strings" - "syscall" - "time" - - "github.com/kelseyhightower/envconfig" - "github.com/ttacon/libphonenumber" - "golang.org/x/sys/unix" - "gopkg.in/yaml.v2" -) - -var ( - regexUUID = regexp.MustCompile("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$") - regexBrief = regexp.MustCompile("^[a-z][a-z0-9\\-]{1,64}$") - regexAppName = regexp.MustCompile("^[a-z][a-z0-9\\_]{1,30}$") - regexExpiration = regexp.MustCompile("^([0-9]+)([mhds])?$") - regexHex = regexp.MustCompile("^[a-zA-F0-9]+$") - indexNames = map[string]bool{ - "custom": true, - "email": true, - "login": true, - "phone": true, - "token": true, - } - consentYesStatuses = map[string]bool{ - "1": true, - "accept": true, - "agree": true, - "approve": true, - "given": true, - "good": true, - "ok": true, - "on": true, - "true": true, - "y": true, - "yes": true, - } - basisTypes = map[string]bool{ - "consent": true, - "contract": true, - "legal-requirement": true, - "legitimate-interest": true, - "public-interest": true, - "vital-interest": true, - } -) - -// Consideration why collection of meta data patch was postpone: -// 1. Databunker is not anti-fraud solution -// 2. GDPR stands for data minimalization. -// 3. Do not store what you actually do not NEED. -/* -var interestingHeaders = []string{"x-forwarded", "x-forwarded-for", "x-coming-from", "via", - "forwarded-for", "forwarded", "client-ip", "user-agent", "cookie", "referer"} - -func getMeta(r *http.Request) string { - headers := bson.M{} - for idx, val := range r.Header { - idx0 := strings.ToLower(idx) - log.Printf("Checking header: %s\n", idx0) - if contains(interestingHeaders, idx0) { - headers[idx] = val[0] - } - } - headersStr, _ := json.Marshal(headers) - meta := fmt.Sprintf(`{"clientip":"%s","headers":%s}`, r.RemoteAddr, headersStr) - log.Printf("Meta: %s\n", meta) - return meta -} -*/ - -func getStringValue(r interface{}) string { - if r == nil { - return "" - } - switch r.(type) { - case string: - return strings.TrimSpace(r.(string)) - case []uint8: - return strings.TrimSpace(string(r.([]uint8))) - case float64: - return strconv.Itoa(int(r.(float64))) - } - return "" -} - -func getIntValue(r interface{}) int { - switch r.(type) { - case int: - return r.(int) - case int32: - return int(r.(int32)) - case float64: - return int(r.(float64)) - } - return 0 -} - -func getInt64Value(records map[string]interface{}, key string) int64 { - if value, ok := records[key]; ok { - switch value.(type) { - case int32: - return int64(value.(int32)) - case int64: - return int64(value.(int64)) - } - } - return 0 -} - -// readConfFile() read configuration file. -func readConfFile(cfg *Config, filepath *string) error { - confFile := "databunker.yaml" - if filepath != nil { - if len(*filepath) > 0 { - confFile = *filepath - } - } - log.Printf("Loading configuration file: %s\n", confFile) - f, err := os.Open(confFile) - if err != nil { - return err - } - decoder := yaml.NewDecoder(f) - err = decoder.Decode(cfg) - if err != nil { - return err - } - return nil -} - -// readEnv() process environment variables. -func readEnv(cfg *Config) error { - err := envconfig.Process("", cfg) - return err -} - -func getArgEnvFileVariable(vname string, masterKeyPtr *string) string { - strvalue := "" - if masterKeyPtr != nil && len(*masterKeyPtr) > 0 { - strvalue = *masterKeyPtr - } else if len(os.Getenv(vname)) > 0 { - strvalue = os.Getenv(vname) - } else if len(os.Getenv(vname+"_FILE")) > 0 { - data, _ := os.ReadFile(os.Getenv(vname + "_FILE")) - strvalue = string(data) - } - return strings.TrimSpace(strvalue) -} - -func hashString(md5Salt []byte, src string) string { - stringToHash := append(md5Salt, []byte(src)...) - hashed := sha256.Sum256(stringToHash) - return base64.StdEncoding.EncodeToString(hashed[:]) -} - -func normalizeConsentStatus(status string) string { - status = strings.ToLower(status) - if consentYesStatuses[status] { - return "yes" - } - return "no" -} - -func normalizeBasisType(status string) string { - status = strings.ToLower(status) - if basisTypes[status] { - return status - } - return "consent" -} - -func normalizeBrief(brief string) string { - return strings.ToLower(brief) -} - -func normalizeEmail(email0 string) string { - email, _ := url.QueryUnescape(email0) - email = strings.ToLower(email) - email = strings.TrimSpace(email) - if email0 != email { - log.Printf("Email before normalization: %s, after: %s\n", email0, email) - } - return email -} - -func normalizePhone(phone string, defaultCountry string) string { - // 4444 is a phone number for testing, no need to normilize it - phone = strings.TrimSpace(phone) - if len(phone) == 0 { - return phone - } - if phone == "4444" { - return "4444" - } - if len(defaultCountry) == 0 { - // https://github.com/ttacon/libphonenumber/blob/master/countrycodetoregionmap.go - defaultCountry = "GB" - } - res, err := libphonenumber.Parse(phone, defaultCountry) - if err != nil { - log.Printf("Failed to parse phone number: %s", phone) - return "" - } - phone = "+" + strconv.Itoa(int(*res.CountryCode)) + strconv.FormatUint(*res.NationalNumber, 10) - return phone -} - -func validateMode(index string) bool { - return indexNames[strings.ToLower(index)] -} - -func parseFields(fields string) []string { - return strings.Split(fields, ",") -} - -// Binary search implementation for a sorted array of strings -func binarySearch(arr []string, target string) bool { - low := 0 - high := len(arr) - 1 - - for low <= high { - mid := (low + high) / 2 - if arr[mid] == target { - return true - } else if arr[mid] < target { - low = mid + 1 - } else { - high = mid - 1 - } - } - return false -} - -func contains(slice []string, item string) bool { - set := make(map[string]bool, len(slice)) - for _, s := range slice { - set[s] = true - } - return set[item] -} - -func atoi(s string) int32 { - var ( - n uint32 - i int - v byte - ) - for ; i < len(s); i++ { - d := s[i] - if '0' <= d && d <= '9' { - v = d - '0' - } else if 'a' <= d && d <= 'z' { - v = d - 'a' + 10 - } else if 'A' <= d && d <= 'Z' { - v = d - 'A' + 10 - } else { - n = 0 - break - } - n *= uint32(10) - n += uint32(v) - } - return int32(n) -} - -func setExpiration(maxExpiration string, userExpiration string) string { - if len(userExpiration) == 0 { - return maxExpiration - } - userExpirationNum, _ := parseExpiration(userExpiration) - maxExpirationNum, _ := parseExpiration(maxExpiration) - if maxExpirationNum == 0 { - maxExpiration = "6m" - maxExpirationNum, _ = parseExpiration(maxExpiration) - } - if userExpirationNum == 0 { - return maxExpiration - } - if userExpirationNum > maxExpirationNum { - return maxExpiration - } - return userExpiration -} - -func parseExpiration0(expiration string) (int32, error) { - match := regexExpiration.FindStringSubmatch(expiration) - // expiration format: 10d, 10h, 10m, 10s - if len(match) != 3 { - e := fmt.Sprintf("failed to parse expiration value: %s", expiration) - return 0, errors.New(e) - } - num := match[1] - format := match[2] - start := int32(0) - switch format { - case "d": // day - start = start + (atoi(num) * 24 * 3600) - case "h": // hour - start = start + (atoi(num) * 3600) - case "m": // month - start = start + (atoi(num) * 24 * 31 * 3600) - case "s": - start = start + (atoi(num)) - } - return start, nil -} - -func parseExpiration(expiration string) (int32, error) { - match := regexExpiration.FindStringSubmatch(expiration) - // expiration format: 10d, 10h, 10m, 10s - if len(match) == 2 { - return atoi(match[1]), nil - } - if len(match) != 3 { - e := fmt.Sprintf("failed to parse expiration value: %s", expiration) - return 0, errors.New(e) - } - num := match[1] - format := match[2] - if len(format) == 0 { - return atoi(num), nil - } - start := int32(time.Now().Unix()) - switch format { - case "d": // day - start = start + (atoi(num) * 24 * 3600) - case "h": // hour - start = start + (atoi(num) * 3600) - case "m": // month - start = start + (atoi(num) * 24 * 31 * 3600) - case "s": - start = start + (atoi(num)) - } - return start, nil -} - -func lockMemory() error { - return unix.Mlockall(syscall.MCL_CURRENT | syscall.MCL_FUTURE) -} - -func isValidUUID(uuidCode string) bool { - return regexUUID.MatchString(uuidCode) -} - -func isValidApp(app string) bool { - return regexAppName.MatchString(app) -} - -func isValidBrief(brief string) bool { - return regexBrief.MatchString(brief) -} - -func isValidHex(hex1 string) bool { - return regexHex.MatchString(hex1) -} - -func isContainer() bool { - //if _, err := os.Stat("/.dockerenv"); err == nil { - // return true - //} - if len(os.Getenv("KUBERNETES_SERVICE_HOST")) > 0 { - return true - } - if _, err := os.Stat("/var/run/secrets/kubernetes.io"); err == nil { - return true - } - return false -} - -// stringPatternMatch looks for basic human patterns like "*", "*abc*", etc... -func stringPatternMatch(pattern string, value string) bool { - if len(pattern) == 0 { - return false - } - if pattern == "*" { - return true - } - if pattern == value { - return true - } - if strings.HasPrefix(pattern, "*") && strings.HasSuffix(pattern, "*") { - pattern = pattern[1 : len(pattern)-1] - if strings.Contains(value, pattern) { - return true - } - return false - } - if strings.HasPrefix(pattern, "*") { - pattern = pattern[1:] - if strings.HasSuffix(value, pattern) { - return true - } - } else if strings.HasSuffix(pattern, "*") { - pattern = pattern[:len(pattern)-1] - if strings.HasPrefix(value, pattern) { - return true - } - } - return false -} - -func returnError(w http.ResponseWriter, r *http.Request, message string, code int, err error, event *auditEvent) { - log.Printf("[%d] %s %s -> Return error\n", code, r.Method, r.URL.Path) - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(code) - fmt.Fprintf(w, `{"status":"error","message":%q}`, message) - if event != nil { - event.Status = "error" - event.Msg = message - if err != nil { - event.Debug = err.Error() - log.Printf("Generate error response: %s, Error: %s\n", message, err.Error()) - } else { - log.Printf("Generate error response: %s\n", message) - } - } - //http.Error(w, http.StatusText(405), 405) -} - -func returnUUID(w http.ResponseWriter, code string) { - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(200) - fmt.Fprintf(w, `{"status":"ok","token":%q}`, code) -} - -func (e mainEnv) enforceAuth(w http.ResponseWriter, r *http.Request, event *auditEvent) string { - /* - for key, value := range r.Header { - fmt.Printf("%s => %s\n", key, value) - } - */ - if token, ok := r.Header["X-Bunker-Token"]; ok { - authResult, err := e.db.checkUserAuthXToken(token[0]) - //fmt.Printf("error in auth? error %s - %s\n", err, token[0]) - if err == nil { - if event != nil { - event.Identity = authResult.name - if authResult.ttype == "login" && authResult.token == event.Record { - return authResult.ttype - } - } - if len(authResult.ttype) > 0 && authResult.ttype != "login" { - return authResult.ttype - } - } - /* - if e.db.checkXtoken(token[0]) == true { - if event != nil { - event.Identity = "admin" - } - return true - } - */ - } - log.Printf("403 Access denied\n") - w.WriteHeader(http.StatusForbidden) - w.Write([]byte("Access denied")) - if event != nil { - event.Status = "error" - event.Msg = "access denied" - } - return "" -} - -func (e mainEnv) enforceAdmin(w http.ResponseWriter, r *http.Request, event *auditEvent) string { - if token, ok := r.Header["X-Bunker-Token"]; ok { - authResult, err := e.db.checkUserAuthXToken(token[0]) - //fmt.Printf("error in auth? error %s - %s\n", err, token[0]) - if err == nil { - if event != nil { - event.Identity = authResult.name - } - if len(authResult.ttype) > 0 && authResult.ttype != "login" { - return authResult.ttype - } - } - } - log.Printf("403 Access denied\n") - w.WriteHeader(http.StatusForbidden) - w.Write([]byte("Access denied")) - return "" -} - -func enforceUUID(w http.ResponseWriter, uuidCode string, event *auditEvent) bool { - if isValidUUID(uuidCode) == false { - //fmt.Printf("405 bad uuid in : %s\n", uuidCode) - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.WriteHeader(405) - fmt.Fprintf(w, `{"status":"error","message":"bad uuid"}`) - if event != nil { - event.Status = "error" - event.Msg = "bad uuid" - } - return false - } - return true -} - -func getJSONPostMap(r *http.Request) (map[string]interface{}, error) { - cType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) - if err != nil { - log.Printf("ignoring empty content-type: %s\n", err) - return nil, nil - } - cType = strings.ToLower(cType) - records := make(map[string]interface{}) - if r.Method == "DELETE" { - // otherwise data is not parsed! - r.Method = "PATCH" - } - body0, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, err - } - body := strings.TrimSpace(string(body0)) - if len(body) < 3 { - return nil, nil - } - if strings.HasPrefix(cType, "application/x-www-form-urlencoded") { - if body[0] == '{' { - return nil, errors.New("wrong content-type, json instead of url encoded data") - } - form, err := url.ParseQuery(body) - if err != nil { - log.Printf("error to parse HTTP data request: %s\n", err) - return nil, err - } - if len(form) == 0 { - return nil, nil - } - for key, value := range form { - //fmt.Printf("data here %s => %s\n", key, value[0]) - records[key] = value[0] - } - } else if strings.HasPrefix(cType, "application/json") { - err = json.Unmarshal([]byte(body), &records) - if err != nil { - log.Printf("Error in json decode %s", err) - return nil, err - } - } else if strings.HasPrefix(cType, "application/xml") { - err = json.Unmarshal([]byte(body), &records) - if err != nil { - log.Printf("Error in xml/json decode %s", err) - return nil, err - } - } else { - log.Printf("Ignore wrong content type: %s", cType) - maxStrLen := 200 - if len(body) < maxStrLen { - maxStrLen = len(body) - } - log.Printf("Body[max 200 chars]: %s", body[0:maxStrLen]) - return nil, nil - } - return records, nil -} - -func getJSONPostData(r *http.Request) ([]byte, error) { - cType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) - if err != nil { - log.Printf("ignoring empty content-type: %s\n", err) - return nil, nil - } - cType = strings.ToLower(cType) - if r.Method == "DELETE" { - // otherwise data is not parsed! - r.Method = "PATCH" - } - body0, err := ioutil.ReadAll(r.Body) - if err != nil { - return nil, err - } - body := strings.TrimSpace(string(body0)) - if len(body) < 3 { - return nil, nil - } - if strings.HasPrefix(cType, "application/x-www-form-urlencoded") { - if body[0] == '{' || body[0] == '[' { - return nil, errors.New("wrong content-type, json instead of url encoded data") - } - form, err := url.ParseQuery(body) - if err != nil { - log.Printf("error in HTTP data request: %s\n", err) - return nil, err - } - if len(form) == 0 { - return nil, nil - } - records := make(map[string]interface{}) - for key, value := range form { - records[key] = value[0] - } - return json.Marshal(records) - } else if strings.HasPrefix(cType, "application/json") || strings.HasPrefix(cType, "application/xml") { - var data interface{} - err := json.Unmarshal([]byte(body), &data) - if err != nil { - return nil, errors.New("error decoding json data") - } - return json.Marshal(data) - } - log.Printf("Ignore wrong content type: %s", cType) - maxStrLen := 200 - if len(body) < maxStrLen { - maxStrLen = len(body) - } - log.Printf("Body[max 200 chars]: %s", body[0:maxStrLen]) - return nil, errors.New("wrong content-type, not a json string") -} - -func getIndexString(val interface{}) string { - switch val.(type) { - case nil: - return "" - case string: - return strings.TrimSpace(val.(string)) - case []uint8: - return strings.TrimSpace(string(val.([]uint8))) - case int: - return strconv.Itoa(val.(int)) - case int64: - return fmt.Sprintf("%v", val.(int64)) - case float64: - return strconv.Itoa(int(val.(float64))) - } - return "" -} - -func getUserJSON(r *http.Request, defaultCountry string) (userJSON, error) { - var result userJSON - records, err := getJSONPostMap(r) - if err != nil { - return result, err - } - if records == nil { - return result, nil - } - - if value, ok := records["login"]; ok { - result.loginIdx = getIndexString(value) - } - if value, ok := records["email"]; ok { - result.emailIdx = normalizeEmail(getIndexString(value)) - } - if value, ok := records["phone"]; ok { - result.phoneIdx = normalizePhone(getIndexString(value), defaultCountry) - } - if value, ok := records["custom"]; ok { - result.customIdx = getIndexString(value) - } - if value, ok := records["token"]; ok { - result.token = value.(string) - } - result.jsonData, err = json.Marshal(records) - return result, err -} - -var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - -func randSeq(n int) string { - b := make([]rune, n) - for i := range b { - b[i] = letters[rand.Intn(len(letters))] - } - return string(b) -} - -var numbers0 = []rune("123456789") -var numbers = []rune("0123456789") - -func randNum(n int) int32 { - b := make([]rune, n) - for i := range b { - b[i] = numbers[rand.Intn(len(numbers))] - } - b[0] = numbers0[rand.Intn(len(numbers0))] - return atoi(string(b)) -} diff --git a/src/utils/utils.go b/src/utils/utils.go index a97b6a17..3bbbf279 100644 --- a/src/utils/utils.go +++ b/src/utils/utils.go @@ -1,4 +1,4 @@ -package main +package utils import ( "crypto/sha256"