Skip to content

Commit fbf54b9

Browse files
jwhitmore-mitrejwhitmore-mitre
authored
Merge pull request #179 from CVEProject/addition_transforms
Additional transforms
2 parents f630a93 + 09dbd27 commit fbf54b9

File tree

2 files changed

+364
-137
lines changed

2 files changed

+364
-137
lines changed

‎schema/v5.0/support/CVE_4_to_5_converter/cve4to5up.py‎

Lines changed: 107 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
scoring_other = {}
4343
invalid_impact_versions = []
4444
requester_map = {}
45+
reference_tag_map = {}
4546
ValidationFailures = {}
4647
cvssErrorList = []
4748
minShortName = 100
@@ -105,7 +106,8 @@ def main(argv):
105106
except:
106107
problemfiles[filepath] = "" + str(sys.exc_info()[0]) + " -- " + str(sys.exc_info()[1]) + " -- "
107108
CVECount += 1
108-
if CVECount % 100 == 0: spinner.next()
109+
# if CVECount % 100 == 0: spinner.next()
110+
if CVECount % 10 == 0: spinner.next()
109111

110112
print('FINISHED processed directory', inputdir)
111113
print('')
@@ -349,6 +351,7 @@ def CVE_Convert(inputfile, outputpath):
349351
global scoring_other
350352
global invalid_impact_versions
351353
global requester_map
354+
global reference_tag_map
352355
global minShortName
353356
global maxShortName
354357
global maxTitle
@@ -358,15 +361,10 @@ def CVE_Convert(inputfile, outputpath):
358361

359362
if len(requester_map) < 1:
360363
getRequesterMap()
361-
362-
''' Not needed if querying IDR by CVE ID
363-
if len(all_users) < 1:
364-
getAllUsers()
365-
# get min and max length of org shortname
366-
for org in all_orgs:
367-
minShortName = min(minShortName, len(all_orgs[org]["short_name"]))
368-
maxShortName = max(maxShortName, len(all_orgs[org]["short_name"]))
369-
'''
364+
365+
if len(reference_tag_map) < 1:
366+
getReferenceTagMap()
367+
370368

371369
with open(inputfile) as json_file:
372370
writeout = False
@@ -388,87 +386,6 @@ def CVE_Convert(inputfile, outputpath):
388386
if i_meta["STATE"] not in keys_used: keys_used[i_meta["STATE"]] = {}
389387
keys_used[i_meta["STATE"]]["CVE_data_meta"] = {}
390388

391-
'''
392-
if "ASSIGNER" in i_meta:
393-
# v4 assigner email converted to orgId before v5 upconvert
394-
# get org info
395-
username = i_meta["ASSIGNER"]
396-
if i_meta["ASSIGNER"] not in all_users:
397-
398-
found = False
399-
# check user mapping first for org name
400-
if username in requester_map:
401-
orgShort = requester_map[username][3]
402-
for org in all_orgs:
403-
if all_orgs[org]["short_name"].casefold() == orgShort.casefold():
404-
o_meta["assignerOrgId"] = all_orgs[org]["UUID"]
405-
o_meta["assignerShortName"] = all_orgs[org]["short_name"]
406-
found = True
407-
break
408-
409-
if not found:
410-
#attempt to locate org from username domain extract
411-
orgShort = username
412-
if '@' in orgShort:
413-
orgShort = orgShort.split('@',1)[1]
414-
if '.' in orgShort:
415-
# strip last .xxxx
416-
orgShort = orgShort.rsplit('.', 1)[0]
417-
# print ("looking for org: "+orgShort)
418-
orgShort = orgShort.lower()
419-
for org in all_orgs:
420-
if all_orgs[org]["short_name"].casefold() == orgShort.casefold():
421-
o_meta["assignerOrgId"] = all_orgs[org]["UUID"]
422-
o_meta["assignerShortName"] = all_orgs[org]["short_name"]
423-
found = True
424-
break
425-
426-
if not found:
427-
if username in requester_map:
428-
orgShort = requester_map[username][3]
429-
for org in all_orgs:
430-
if all_orgs[org]["short_name"].casefold() == orgShort.casefold():
431-
o_meta["assignerOrgId"] = all_orgs[org]["UUID"]
432-
o_meta["assignerShortName"] = all_orgs[org]["short_name"]
433-
found = True
434-
break
435-
else:
436-
# look for email half
437-
if '@' in username:
438-
userShort = username.split('@',1)[1]
439-
for user in requester_map:
440-
tuser = user
441-
if '@' in tuser:
442-
tuser = user.split('@',1)[1]
443-
if userShort.casefold() == tuser.casefold():
444-
orgShort = requester_map[user][3]
445-
for org in all_orgs:
446-
if all_orgs[org]["short_name"].casefold() == orgShort.casefold():
447-
o_meta["assignerOrgId"] = all_orgs[org]["UUID"]
448-
o_meta["assignerShortName"] = all_orgs[org]["short_name"]
449-
found = True
450-
break
451-
# end for org
452-
if found: break
453-
# end for user
454-
else:
455-
print("found username that is not an email: "+ str(username) )
456-
# end if - user_map else parsed email magic
457-
458-
if not found:
459-
# default to MITRE and CNAofLR
460-
if username not in defaulted_users:
461-
defaulted_users[username] = []
462-
defaulted_users[username].append(i_meta["ID"])
463-
username = "DEFAULT"
464-
o_meta["assignerOrgId"] = all_users[username]["org_UUID"]
465-
o_meta["assignerShortName"] = all_users[username]["org_short_name"]
466-
else:
467-
o_meta["assignerOrgId"] = all_users[username]["org_UUID"]
468-
o_meta["assignerShortName"] = all_users[username]["org_short_name"]
469-
470-
# print("in = " + i_meta["ASSIGNER"] + " out = " +o_meta["assignerShortName"])
471-
'''
472389

473390
if "STATE" in i_meta:
474391
if i_meta["STATE"] == 'RESERVED':
@@ -533,14 +450,12 @@ def CVE_Convert(inputfile, outputpath):
533450
else:
534451
raise MissingRequiredPropertyValue(inputfile, "CVE_data_meta no STATE")
535452
except Exception as e:
536-
# print("test 5")
537453
print( str(e) )
538454
if type(e) is not MissingRequiredPropertyValue:
539455
raise MissingRequiredPropertyValue(inputfile, "CVE_data_meta structure error")
540456
else:
541457
raise e
542458

543-
# o_meta["dateUpdated"] = datetime.date.today().strftime("%Y-%m-%d")
544459
o_meta["dateUpdated"] = str(datetime.datetime.combine(datetime.date.today(), datetime.datetime.min.time()).isoformat())
545460

546461
jout["cveMetadata"] = o_meta
@@ -646,6 +561,7 @@ def CVE_Convert(inputfile, outputpath):
646561
if "version" in vd_pd and "version_data" in vd_pd["version"]:
647562
v_agg_hash = {}
648563
v_agg_list = {}
564+
product_name = vd_pd["product_name"]
649565
for pd_vd in vd_pd["version"]["version_data"]:
650566
if not "version_value" in pd_vd:
651567
# throw invalid version_data, must have version_value value
@@ -658,37 +574,39 @@ def CVE_Convert(inputfile, outputpath):
658574
v_agg_list[platform] = []
659575
vn_hash = v_agg_hash[platform]
660576
v_list = v_agg_list[platform]
661-
if "version_name" in pd_vd: # vulnogram generated
577+
if "version_name" in pd_vd: # vulnogram generated
662578
vn = pd_vd["version_name"]
663-
[vstatus, va] = convert_VA(pd_vd)
664-
if va == '=':
665-
v_list.append(nonEmpty(eq_version(pd_vd, vstatus)))
666-
elif vn in vn_hash:
667-
if va == '<':
668-
vstatus = negate(vstatus)
669-
if va == '<=':
670-
vstatus = negate(vstatus)
671-
pd_vd["version_value"] = pd_vd["version_value"] + ' +1'
579+
if product_name.casefold() is not vn.casefold():
580+
[vstatus, va] = convert_VA(pd_vd)
581+
if va == '=':
582+
v_list.append(nonEmpty(eq_version(pd_vd, vstatus)))
583+
elif vn in vn_hash:
584+
if va == '<':
585+
vstatus = negate(vstatus)
586+
if va == '<=':
587+
vstatus = negate(vstatus)
588+
pd_vd["version_value"] = pd_vd["version_value"] + ' +1'
589+
else:
590+
if not "changes" in vn_hash[vn]:
591+
vn_hash[vn]["changes"] = []
592+
chg = {
593+
"at": pd_vd["version_value"],
594+
"status": vstatus
595+
}
596+
if chg not in vn_hash[vn]["changes"]:
597+
vn_hash[vn]["changes"].append(chg)
598+
elif va == '<':
599+
vn_hash[vn] = nonEmpty(l_version(pd_vd, vstatus))
600+
elif va == '<=':
601+
vn_hash[vn] = nonEmpty(le_version(pd_vd, vstatus))
672602
else:
673-
if not "changes" in vn_hash[vn]:
674-
vn_hash[vn]["changes"] = []
675-
chg = {
676-
"at": pd_vd["version_value"],
677-
"status": vstatus
603+
vn_hash[vn] = {
604+
"version": pd_vd["version_value"],
605+
"status": vstatus,
606+
"lessThan": pd_vd["version_name"] + '*',
607+
"versionType": "custom"
678608
}
679-
if chg not in vn_hash[vn]["changes"]:
680-
vn_hash[vn]["changes"].append(chg)
681-
elif va == '<':
682-
vn_hash[vn] = nonEmpty(l_version(pd_vd, vstatus))
683-
elif va == '<=':
684-
vn_hash[vn] = nonEmpty(le_version(pd_vd, vstatus))
685-
else:
686-
vn_hash[vn] = {
687-
"version": pd_vd["version_value"],
688-
"status": vstatus,
689-
"lessThan": pd_vd["version_name"] + '*',
690-
"versionType": "custom"
691-
}
609+
# end if product_name is not version_name
692610
else:
693611
[vstatus, va] = convert_VA(pd_vd)
694612
version_value = pd_vd["version_value"]
@@ -740,6 +658,7 @@ def CVE_Convert(inputfile, outputpath):
740658
# check for blank version and defailt to "unspecified"
741659
#if len(version_item["version"]) < 1:
742660
# version_item["version"] = "unspecified"
661+
# end if version_name in pd_vd
743662

744663
for platform in v_agg_hash:
745664
# build affected item here:
@@ -803,9 +722,17 @@ def CVE_Convert(inputfile, outputpath):
803722
if "refsource" in i_ref:
804723
if "tags" not in o_ref:
805724
o_ref["tags"] = []
725+
726+
# convert to new reference tags
727+
v5Tag_value = getV5ReferenceTagValue(i_ref["refsource"])
728+
if v5Tag_value not in o_ref["tags"]:
729+
o_ref["tags"].append(v5Tag_value)
730+
731+
# preserve legacy tag value
806732
refSourceTag = "x_refsource_"+i_ref["refsource"]
807733
if refSourceTag not in o_ref["tags"]:
808734
o_ref["tags"].append(refSourceTag)
735+
809736
if "url" in i_ref: o_ref["url"] = i_ref["url"]
810737

811738
# decode then encode URL, to clear issue with AJV URL validations
@@ -821,17 +748,34 @@ def CVE_Convert(inputfile, outputpath):
821748
keys_used["PUBLIC"]["credit"] = ""
822749
if isinstance(data["credit"], list):
823750
for i_credit in data["credit"]:
824-
o_credit = {}
825-
if "lang" in i_credit and "value" in i_credit:
826-
o_credit["lang"] = lang_code_2_from_3(i_credit["lang"])
751+
if isinstance(i_credit, dict):
752+
o_credit = {}
753+
if "lang" in i_credit and "value" in i_credit:
754+
o_credit["lang"] = lang_code_2_from_3(i_credit["lang"])
755+
else:
756+
o_credit["lang"] = "en"
757+
758+
if "value" in i_credit:
759+
if "credits" not in o_cna:
760+
o_cna["credits"] = []
761+
o_credit["value"] = i_credit["value"]
762+
o_cna["credits"].append(o_credit)
763+
elif isinstance(i_credit, list):
764+
for citem in i_credit:
765+
o_credit = {}
766+
o_credit["lang"] = "en"
767+
if "credits" not in o_cna:
768+
o_cna["credits"] = []
769+
o_credit["value"] = citem
770+
o_cna["credits"].append(o_credit)
827771
else:
772+
o_credit = {}
828773
o_credit["lang"] = "en"
829-
830-
if "value" in i_credit:
774+
o_credit["value"] = i_credit
831775
if "credits" not in o_cna:
832776
o_cna["credits"] = []
833-
o_credit["value"] = i_credit["value"]
834777
o_cna["credits"].append(o_credit)
778+
835779
else:
836780
# convert value content to string
837781
o_cna["credits"] = []
@@ -1487,7 +1431,7 @@ def getAllUsers():
14871431

14881432

14891433

1490-
def getIDRInfo(cveId, delay=20, retry=0):
1434+
def getIDRInfo(cveId, delay=300, retry=0):
14911435
IDR_URL = settings.AWG_IDR_SERVICE_URL + '/cve-id/' + cveId
14921436
idr_params = {}
14931437
data = None
@@ -1499,22 +1443,23 @@ def getIDRInfo(cveId, delay=20, retry=0):
14991443
if idr_result and idr_result.startswith("{"):
15001444
data = json.loads(idr_result)
15011445
else:
1502-
if retry < 10:
1503-
# print("delay for: "+ str(delay))
1446+
if retry < 14:
1447+
print("delaying for: "+ str(delay) + " -- on -- " + cveId)
15041448
time.sleep(delay)
1505-
data = getIDRInfo(cveId, delay*2, retry+1)
1449+
data = getIDRInfo(cveId, delay, retry+1)
15061450
else:
1507-
print("Record Issue - URL - " + IDR_URL)
1451+
print("Record Timeout Issue - URL - " + IDR_URL)
15081452
# print(str(idr_result))
15091453
except Exception as e:
1510-
if retry < 10:
1511-
if delay > 179:
1512-
print("exception delay for: " + str(delay))
1454+
if retry < 14:
1455+
# if delay > 179:
1456+
print("Exception delay for: " + str(delay))
1457+
print(" --- " + IDR_URL)
15131458
time.sleep(delay)
1514-
data = getIDRInfo(cveId, delay*2, retry+1)
1459+
data = getIDRInfo(cveId, delay, retry+1)
15151460
else:
15161461
# print(str(idr_result))
1517-
print("Exception -- URL - " + IDR_URL)
1462+
print("Exception Failed -- get IDR info -- URL - " + IDR_URL)
15181463
print(str(e))
15191464
raise e
15201465
return data
@@ -1570,6 +1515,31 @@ def getRequesterMap():
15701515
return True
15711516

15721517

1518+
def getReferenceTagMap():
1519+
global reference_tag_map
1520+
1521+
if len(reference_tag_map) < 1 :
1522+
with open("ref_tag_map.json") as ref_tag_file:
1523+
reference_tag_map = json.load(ref_tag_file)
1524+
return True
1525+
1526+
1527+
def getV5ReferenceTagValue(v4Tag):
1528+
global reference_tag_map
1529+
v5Tag = "related"
1530+
v4Test = v4Tag.casefold()
1531+
refhit = False
1532+
for tagMap in reference_tag_map["referenceMaps"]:
1533+
if v4Test == tagMap["v4"].casefold():
1534+
v5Tag = tagMap["v5"]
1535+
refhit = True
1536+
break
1537+
if not refhit:
1538+
print("Missed Ref Tag: " + v4Tag)
1539+
1540+
return v5Tag
1541+
1542+
15731543
def call_idr_service(action, req_header, IDR_URL, params=None, content=None):
15741544
"""
15751545
:param action: GET, POST, ...

0 commit comments

Comments
 (0)