Skip to content

feat(recon): retain graph nodes across reingest #140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gbxbgbl wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat(recon): retain graph nodes across reingest #140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions graph_db/mixins/base_mixin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
"""

import os
import uuid
from datetime import datetime, timezone
from pathlib import Path

from neo4j import GraphDatabase
Expand All @@ -25,10 +27,41 @@ def __init__(self, uri=None, user=None, password=None):
self.uri = uri or os.getenv("NEO4J_URI", "bolt://localhost:7687")
self.user = user or os.getenv("NEO4J_USER")
self.password = password or os.getenv("NEO4J_PASSWORD")
self._init_recon_job_context()
self.driver = GraphDatabase.driver(self.uri, auth=(self.user, self.password))
with self.driver.session() as session:
init_schema(session)

def _init_recon_job_context(self):
self.recon_job_started_at = os.getenv("RECON_JOB_STARTED_AT")
if not self.recon_job_started_at:
self.recon_job_started_at = (
datetime.now(timezone.utc)
.replace(microsecond=0)
.isoformat()
.replace("+00:00", "Z")
)

self.recon_job_id = os.getenv("RECON_JOB_ID")
if not self.recon_job_id:
stamp = self.recon_job_started_at.replace("-", "").replace(":", "")
stamp = stamp.replace("+0000", "Z").replace(".", "")
self.recon_job_id = f"adhoc-{stamp[:16]}-{uuid.uuid4().hex[:8]}"

def _recon_job_params(self) -> dict:
return {
"recon_job_id": self.recon_job_id,
"recon_job_started_at": self.recon_job_started_at,
}

def _node_seen_set_clause(self, alias: str) -> str:
return (
f"{alias}.first_seen = coalesce({alias}.first_seen, $recon_job_started_at), "
f"{alias}.first_seen_job_id = coalesce({alias}.first_seen_job_id, $recon_job_id), "
f"{alias}.last_seen = $recon_job_started_at, "
f"{alias}.last_seen_job_id = $recon_job_id"
)

def close(self):
self.driver.close()

Expand Down
91 changes: 76 additions & 15 deletions graph_db/mixins/graphql_mixin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,15 @@ def _is_confirmed_graphql(info: dict) -> bool:
project_id: $project_id
})
ON CREATE SET bu.source = 'graphql_scan',
bu.updated_at = datetime()
bu.updated_at = datetime(),
bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at),
bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id),
bu.last_seen = $recon_job_started_at,
bu.last_seen_job_id = $recon_job_id
ON MATCH SET bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at),
bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id),
bu.last_seen = $recon_job_started_at,
bu.last_seen_job_id = $recon_job_id
WITH bu
MERGE (e:Endpoint {
path: $path,
Expand All @@ -197,16 +205,25 @@ def _is_confirmed_graphql(info: dict) -> bool:
project_id: $project_id
})
ON CREATE SET e.source = 'graphql_scan',
e.created_at = datetime()
SET e += $props
e.created_at = datetime(),
e.first_seen = coalesce(e.first_seen, $recon_job_started_at),
e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id),
e.last_seen = $recon_job_started_at,
e.last_seen_job_id = $recon_job_id
SET e += $props,
e.first_seen = coalesce(e.first_seen, $recon_job_started_at),
e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id),
e.last_seen = $recon_job_started_at,
e.last_seen_job_id = $recon_job_id
MERGE (bu)-[:HAS_ENDPOINT]->(e)
RETURN e.path as path, e.is_graphql as was_graphql
""",
path=path,
baseurl=baseurl,
user_id=user_id,
project_id=project_id,
props=graphql_props
props=graphql_props,
**self._recon_job_params(),
)

record = result.single()
Expand Down Expand Up @@ -320,13 +337,18 @@ def _is_confirmed_graphql(info: dict) -> bool:
user_id: $user_id,
project_id: $project_id
})
SET v += $props
SET v += $props,
v.first_seen = coalesce(v.first_seen, $recon_job_started_at),
v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id),
v.last_seen = $recon_job_started_at,
v.last_seen_job_id = $recon_job_id
RETURN v.id as id
""",
id=vuln_id,
user_id=user_id,
project_id=project_id,
props=vuln_props
props=vuln_props,
**self._recon_job_params(),
)

record = result.single()
Expand All @@ -353,7 +375,15 @@ def _is_confirmed_graphql(info: dict) -> bool:
project_id: $project_id
})
ON CREATE SET bu.source = 'graphql_scan',
bu.updated_at = datetime()
bu.updated_at = datetime(),
bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at),
bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id),
bu.last_seen = $recon_job_started_at,
bu.last_seen_job_id = $recon_job_id
ON MATCH SET bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at),
bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id),
bu.last_seen = $recon_job_started_at,
bu.last_seen_job_id = $recon_job_id
MERGE (e:Endpoint {
path: $path,
method: 'POST',
Expand All @@ -363,15 +393,24 @@ def _is_confirmed_graphql(info: dict) -> bool:
})
ON CREATE SET e.source = 'graphql_scan',
e.is_graphql = true,
e.created_at = datetime()
e.created_at = datetime(),
e.first_seen = coalesce(e.first_seen, $recon_job_started_at),
e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id),
e.last_seen = $recon_job_started_at,
e.last_seen_job_id = $recon_job_id
ON MATCH SET e.first_seen = coalesce(e.first_seen, $recon_job_started_at),
e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id),
e.last_seen = $recon_job_started_at,
e.last_seen_job_id = $recon_job_id
MERGE (bu)-[:HAS_ENDPOINT]->(e)
MERGE (e)-[:HAS_VULNERABILITY]->(v)
""",
vuln_id=vuln_id,
path=path,
baseurl=baseurl,
user_id=user_id,
project_id=project_id
project_id=project_id,
**self._recon_job_params(),
)
stats["relationships_created"] += 1

Expand Down Expand Up @@ -403,13 +442,18 @@ def _is_confirmed_graphql(info: dict) -> bool:
user_id: $user_id,
project_id: $project_id
})
SET v += $props
SET v += $props,
v.first_seen = coalesce(v.first_seen, $recon_job_started_at),
v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id),
v.last_seen = $recon_job_started_at,
v.last_seen_job_id = $recon_job_id
RETURN v.id as id
""",
id=sensitive_vuln_id,
user_id=user_id,
project_id=project_id,
props=sensitive_props
props=sensitive_props,
**self._recon_job_params(),
)

if result.single():
Expand All @@ -429,7 +473,15 @@ def _is_confirmed_graphql(info: dict) -> bool:
project_id: $project_id
})
ON CREATE SET bu.source = 'graphql_scan',
bu.updated_at = datetime()
bu.updated_at = datetime(),
bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at),
bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id),
bu.last_seen = $recon_job_started_at,
bu.last_seen_job_id = $recon_job_id
ON MATCH SET bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at),
bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id),
bu.last_seen = $recon_job_started_at,
bu.last_seen_job_id = $recon_job_id
MERGE (e:Endpoint {
path: $path,
method: 'POST',
Expand All @@ -439,15 +491,24 @@ def _is_confirmed_graphql(info: dict) -> bool:
})
ON CREATE SET e.source = 'graphql_scan',
e.is_graphql = true,
e.created_at = datetime()
e.created_at = datetime(),
e.first_seen = coalesce(e.first_seen, $recon_job_started_at),
e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id),
e.last_seen = $recon_job_started_at,
e.last_seen_job_id = $recon_job_id
ON MATCH SET e.first_seen = coalesce(e.first_seen, $recon_job_started_at),
e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id),
e.last_seen = $recon_job_started_at,
e.last_seen_job_id = $recon_job_id
MERGE (bu)-[:HAS_ENDPOINT]->(e)
MERGE (e)-[:HAS_VULNERABILITY]->(v)
""",
vuln_id=sensitive_vuln_id,
path=path,
baseurl=baseurl,
user_id=user_id,
project_id=project_id
project_id=project_id,
**self._recon_job_params(),
)
stats["relationships_created"] += 1

Expand All @@ -464,4 +525,4 @@ def _is_confirmed_graphql(info: dict) -> bool:
for error in stats["errors"][:5]: # Show first 5 errors
print(f"[!][graph-db] {error}")

return stats
return stats
Loading