Skip to content

Commit 6d12798

Browse files
natebwangsutnatebwangsut
authored
feat: added GitLab secret detection report parser (DefectDojo#4605)
* feat: added GitLab secret detection report parser * fix: Flake8 compliance * fix: remove entry from test_type.json * fix: change severity on parser * fix: update findings description field for raw_source_code * fix: add unique_id_from_tool for findings * fix: remove autoimported module * fix: typo * test: added unittests for unique_id_from_tool field * fix: Flake8 compliant * test: add testing for findings date * fix: datetime bug parser
1 parent b5eb7e5 commit 6d12798

File tree

6 files changed

+306
-0
lines changed

6 files changed

+306
-0
lines changed

‎dojo/tools/gitlab_secret_detection_report/__init__.py‎

Whitespace-only changes.

‎dojo/tools/gitlab_secret_detection_report/parser.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import json
2+
from datetime import datetime
3+
from dojo.models import Finding
4+
5+
6+
class GitlabSecretDetectionReportParser(object):
7+
"""
8+
GitLab's secret detection report
9+
See more: https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/blob/master/dist/secret-detection-report-format.json
10+
"""
11+
12+
def get_scan_types(self):
13+
return ["GitLab Secret Detection Report"]
14+
15+
def get_label_for_scan_types(self, scan_type):
16+
return "GitLab Secret Detection Report"
17+
18+
def get_description_for_scan_types(self, scan_type):
19+
return "GitLab Secret Detection Report file can be imported in JSON format (option --json)."
20+
21+
def get_findings(self, file, test):
22+
# Load JSON data from uploaded file
23+
data = json.load(file)
24+
25+
findings = []
26+
27+
# This is required by schema - it won't be null / undefined
28+
date = datetime.strptime(data["scan"]["end_time"], "%Y-%m-%dT%H:%M:%S")
29+
30+
# Vulnerabilities is stored on vulnerabilities key
31+
vulnerabilities = data["vulnerabilities"]
32+
for vulnerability in vulnerabilities:
33+
title = vulnerability["message"]
34+
description = vulnerability["description"]
35+
severity = self.normalise_severity(vulnerability["severity"])
36+
location = vulnerability["location"]
37+
finding = Finding(
38+
test=test,
39+
title=title,
40+
description=description,
41+
date=date,
42+
severity=severity,
43+
static_finding=True,
44+
dynamic_finding=False,
45+
unique_id_from_tool=vulnerability["id"],
46+
)
47+
48+
if "file" in location:
49+
finding.file_path = location["file"]
50+
if "start_line" in location:
51+
finding.line = int(location["start_line"])
52+
if "raw_source_code_extract" in vulnerability:
53+
finding.description += "\n" + vulnerability["raw_source_code_extract"]
54+
55+
findings.append(finding)
56+
return findings
57+
58+
def normalise_severity(self, severity):
59+
"""
60+
Normalise GitLab's severity to DefectDojo's
61+
(Critical, High, Medium, Low, Unknown, Info) -> (Critical, High, Medium, Low, Info)
62+
"""
63+
if severity == "Unknown":
64+
return "Info"
65+
return severity

‎dojo/unittests/scans/gitlab_secret_detection_report/gitlab_secret_detection_report_0_vuln.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"version": "14.0.0",
3+
"vulnerabilities": [],
4+
"remediations": [],
5+
"scan": {
6+
"scanner": {
7+
"id": "gitleaks",
8+
"name": "Gitleaks",
9+
"url": "https://github.com/zricethezav/gitleaks",
10+
"vendor": {
11+
"name": "GitLab"
12+
},
13+
"version": "v7.5.0"
14+
},
15+
"type": "secret_detection",
16+
"start_time": "2021-06-02T09:13:09",
17+
"end_time": "2021-06-02T09:13:09",
18+
"status": "success"
19+
}
20+
}

‎dojo/unittests/scans/gitlab_secret_detection_report/gitlab_secret_detection_report_1_vuln.json‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{
2+
"version": "14.0.0",
3+
"vulnerabilities": [
4+
{
5+
"id": "714ed3e4e289ad35a089e0a888e8d0120b6a6083b1090a189cbc6a3227396240",
6+
"category": "secret_detection",
7+
"name": "AWS",
8+
"message": "AWS detected; please remove and revoke it if this is a leak.",
9+
"description": "AWS",
10+
"cve": "README.md:1a5d44a2dca19669d72edf4c4f1c27c4c1ca4b4408fbb17f6ce4ad452d78ddb3:AWS",
11+
"severity": "Critical",
12+
"confidence": "Unknown",
13+
"raw_source_code_extract": "AKIAIOSFODNN7EXAMPLE",
14+
"scanner": {
15+
"id": "gitleaks",
16+
"name": "Gitleaks"
17+
},
18+
"location": {
19+
"file": "README.md",
20+
"commit": {
21+
"date": "0001-01-01T00:00:00Z",
22+
"sha": "0000000"
23+
},
24+
"start_line": 5,
25+
"end_line": 5
26+
},
27+
"identifiers": [
28+
{
29+
"type": "gitleaks_rule_id",
30+
"name": "Gitleaks rule ID AWS",
31+
"value": "AWS"
32+
}
33+
]
34+
}
35+
],
36+
"remediations": [],
37+
"scan": {
38+
"scanner": {
39+
"id": "gitleaks",
40+
"name": "Gitleaks",
41+
"url": "https://github.com/zricethezav/gitleaks",
42+
"vendor": {
43+
"name": "GitLab"
44+
},
45+
"version": "v7.5.0"
46+
},
47+
"type": "secret_detection",
48+
"start_time": "2021-06-02T09:13:09",
49+
"end_time": "2021-06-02T09:13:09",
50+
"status": "success"
51+
}
52+
}

‎dojo/unittests/scans/gitlab_secret_detection_report/gitlab_secret_detection_report_3_vuln.json‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
{
2+
"version": "14.0.0",
3+
"vulnerabilities": [
4+
{
5+
"id": "918e2cdfda408c352ee1f9b1fd148926578a8cfec32a82cd0cd21d4aded6e85e",
6+
"category": "secret_detection",
7+
"name": "SSH private key",
8+
"message": "SSH private key detected; please remove and revoke it if this is a leak.",
9+
"description": "SSH private key",
10+
"cve": "README.md:03d104c669e3c7b6be7f989db8b12c8b910d3be8c1e2a73c9369d3cc0ba803b5:SSH private key",
11+
"severity": "Critical",
12+
"confidence": "Unknown",
13+
"raw_source_code_extract": "-----BEGIN OPENSSH PRIVATE KEY-----",
14+
"scanner": {
15+
"id": "gitleaks",
16+
"name": "Gitleaks"
17+
},
18+
"location": {
19+
"file": "README.md",
20+
"commit": {
21+
"date": "0001-01-01T00:00:00Z",
22+
"sha": "0000000"
23+
},
24+
"start_line": 20,
25+
"end_line": 20
26+
},
27+
"identifiers": [
28+
{
29+
"type": "gitleaks_rule_id",
30+
"name": "Gitleaks rule ID SSH private key",
31+
"value": "SSH private key"
32+
}
33+
]
34+
},
35+
{
36+
"id": "5f2fe26d5029737fcd2d131b5f881ddb32edf7ebc003dfab8b5bcd4f05640f98",
37+
"category": "secret_detection",
38+
"name": "AWS",
39+
"message": "AWS detected; please remove and revoke it if this is a leak.",
40+
"description": "AWS",
41+
"cve": "README.md:1a5d44a2dca19669d72edf4c4f1c27c4c1ca4b4408fbb17f6ce4ad452d78ddb3:AWS",
42+
"severity": "Critical",
43+
"confidence": "Unknown",
44+
"raw_source_code_extract": "AKIAIOSFODNN7EXAMPLE",
45+
"scanner": {
46+
"id": "gitleaks",
47+
"name": "Gitleaks"
48+
},
49+
"location": {
50+
"file": "README.md",
51+
"commit": {
52+
"date": "0001-01-01T00:00:00Z",
53+
"sha": "0000000"
54+
},
55+
"start_line": 7,
56+
"end_line": 7
57+
},
58+
"identifiers": [
59+
{
60+
"type": "gitleaks_rule_id",
61+
"name": "Gitleaks rule ID AWS",
62+
"value": "AWS"
63+
}
64+
]
65+
},
66+
{
67+
"id": "b472ce0c4949cc5e22fc0193978b8eab4a7bafe5cd3d23c8de217d11a59431c5",
68+
"category": "secret_detection",
69+
"name": "Password in URL",
70+
"message": "Password in URL detected; please remove and revoke it if this is a leak.",
71+
"description": "Password in URL",
72+
"cve": "README.md:9aa1da8f23a3e99f2b894638b0d6c584b1613075eca0f74d79de0c7bd07e150b:Password in URL",
73+
"severity": "Critical",
74+
"confidence": "Unknown",
75+
"raw_source_code_extract": "https://random:[email protected]/path",
76+
"scanner": {
77+
"id": "gitleaks",
78+
"name": "Gitleaks"
79+
},
80+
"location": {
81+
"file": "README.md",
82+
"commit": {
83+
"date": "0001-01-01T00:00:00Z",
84+
"sha": "0000000"
85+
},
86+
"start_line": 14,
87+
"end_line": 14
88+
},
89+
"identifiers": [
90+
{
91+
"type": "gitleaks_rule_id",
92+
"name": "Gitleaks rule ID Password in URL",
93+
"value": "Password in URL"
94+
}
95+
]
96+
}
97+
],
98+
"remediations": [],
99+
"scan": {
100+
"scanner": {
101+
"id": "gitleaks",
102+
"name": "Gitleaks",
103+
"url": "https://github.com/zricethezav/gitleaks",
104+
"vendor": {
105+
"name": "GitLab"
106+
},
107+
"version": "v7.5.0"
108+
},
109+
"type": "secret_detection",
110+
"start_time": "2021-06-02T09:25:50",
111+
"end_time": "2021-06-02T09:25:50",
112+
"status": "success"
113+
}
114+
}

‎dojo/unittests/tools/test_gitlab_secret_detection_report_parser.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
from datetime import datetime
2+
from django.test import TestCase
3+
from dojo.tools.gitlab_secret_detection_report.parser import (
4+
GitlabSecretDetectionReportParser,
5+
)
6+
from dojo.models import Test
7+
8+
9+
class TestGitlabSecretDetectionReportParser(TestCase):
10+
def test_gitlab_secret_detection_report_parser_with_no_vuln_has_no_findings(self):
11+
testfile = open(
12+
"dojo/unittests/scans/gitlab_secret_detection_report/gitlab_secret_detection_report_0_vuln.json"
13+
)
14+
parser = GitlabSecretDetectionReportParser()
15+
findings = parser.get_findings(testfile, Test())
16+
testfile.close()
17+
self.assertEqual(0, len(findings))
18+
19+
def test_gitlab_secret_detection_report_parser_with_one_vuln_has_one_findings(
20+
self,
21+
):
22+
testfile = open(
23+
"dojo/unittests/scans/gitlab_secret_detection_report/gitlab_secret_detection_report_1_vuln.json"
24+
)
25+
parser = GitlabSecretDetectionReportParser()
26+
findings = parser.get_findings(testfile, Test())
27+
testfile.close()
28+
for finding in findings:
29+
for endpoint in finding.unsaved_endpoints:
30+
endpoint.clean()
31+
first_finding = findings[0]
32+
self.assertEqual(1, len(findings))
33+
self.assertEqual(datetime(2021, 6, 2, 9, 13, 9), first_finding.date)
34+
self.assertEqual(5, first_finding.line)
35+
self.assertEqual("Critical", first_finding.severity)
36+
self.assertEqual("README.md", first_finding.file_path)
37+
self.assertEqual("AWS\nAKIAIOSFODNN7EXAMPLE", first_finding.description)
38+
self.assertEqual(
39+
"714ed3e4e289ad35a089e0a888e8d0120b6a6083b1090a189cbc6a3227396240",
40+
first_finding.unique_id_from_tool,
41+
)
42+
43+
def test_gitlab_secret_detection_report_parser_with_many_vuln_has_many_findings(
44+
self,
45+
):
46+
testfile = open(
47+
"dojo/unittests/scans/gitlab_secret_detection_report/gitlab_secret_detection_report_3_vuln.json"
48+
)
49+
parser = GitlabSecretDetectionReportParser()
50+
findings = parser.get_findings(testfile, Test())
51+
testfile.close()
52+
for finding in findings:
53+
for endpoint in finding.unsaved_endpoints:
54+
endpoint.clean()
55+
self.assertEqual(3, len(findings))

0 commit comments

Comments
 (0)