diff --git a/graph_db/mixins/base_mixin.py b/graph_db/mixins/base_mixin.py index f5f38fe8..1499d562 100644 --- a/graph_db/mixins/base_mixin.py +++ b/graph_db/mixins/base_mixin.py @@ -9,6 +9,8 @@ """ import os +import uuid +from datetime import datetime, timezone from pathlib import Path from neo4j import GraphDatabase @@ -25,10 +27,41 @@ def __init__(self, uri=None, user=None, password=None): self.uri = uri or os.getenv("NEO4J_URI", "bolt://localhost:7687") self.user = user or os.getenv("NEO4J_USER") self.password = password or os.getenv("NEO4J_PASSWORD") + self._init_recon_job_context() self.driver = GraphDatabase.driver(self.uri, auth=(self.user, self.password)) with self.driver.session() as session: init_schema(session) + def _init_recon_job_context(self): + self.recon_job_started_at = os.getenv("RECON_JOB_STARTED_AT") + if not self.recon_job_started_at: + self.recon_job_started_at = ( + datetime.now(timezone.utc) + .replace(microsecond=0) + .isoformat() + .replace("+00:00", "Z") + ) + + self.recon_job_id = os.getenv("RECON_JOB_ID") + if not self.recon_job_id: + stamp = self.recon_job_started_at.replace("-", "").replace(":", "") + stamp = stamp.replace("+0000", "Z").replace(".", "") + self.recon_job_id = f"adhoc-{stamp[:16]}-{uuid.uuid4().hex[:8]}" + + def _recon_job_params(self) -> dict: + return { + "recon_job_id": self.recon_job_id, + "recon_job_started_at": self.recon_job_started_at, + } + + def _node_seen_set_clause(self, alias: str) -> str: + return ( + f"{alias}.first_seen = coalesce({alias}.first_seen, $recon_job_started_at), " + f"{alias}.first_seen_job_id = coalesce({alias}.first_seen_job_id, $recon_job_id), " + f"{alias}.last_seen = $recon_job_started_at, " + f"{alias}.last_seen_job_id = $recon_job_id" + ) + def close(self): self.driver.close() diff --git a/graph_db/mixins/graphql_mixin.py b/graph_db/mixins/graphql_mixin.py index fc2a9383..0779864a 100644 --- a/graph_db/mixins/graphql_mixin.py +++ b/graph_db/mixins/graphql_mixin.py @@ -187,7 +187,15 @@ def _is_confirmed_graphql(info: dict) -> bool: project_id: $project_id }) ON CREATE SET bu.source = 'graphql_scan', - bu.updated_at = datetime() + bu.updated_at = datetime(), + bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at), + bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id), + bu.last_seen = $recon_job_started_at, + bu.last_seen_job_id = $recon_job_id + ON MATCH SET bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at), + bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id), + bu.last_seen = $recon_job_started_at, + bu.last_seen_job_id = $recon_job_id WITH bu MERGE (e:Endpoint { path: $path, @@ -197,8 +205,16 @@ def _is_confirmed_graphql(info: dict) -> bool: project_id: $project_id }) ON CREATE SET e.source = 'graphql_scan', - e.created_at = datetime() - SET e += $props + e.created_at = datetime(), + e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id + SET e += $props, + e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id MERGE (bu)-[:HAS_ENDPOINT]->(e) RETURN e.path as path, e.is_graphql as was_graphql """, @@ -206,7 +222,8 @@ def _is_confirmed_graphql(info: dict) -> bool: baseurl=baseurl, user_id=user_id, project_id=project_id, - props=graphql_props + props=graphql_props, + **self._recon_job_params(), ) record = result.single() @@ -320,13 +337,18 @@ def _is_confirmed_graphql(info: dict) -> bool: user_id: $user_id, project_id: $project_id }) - SET v += $props + SET v += $props, + v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id), + v.last_seen = $recon_job_started_at, + v.last_seen_job_id = $recon_job_id RETURN v.id as id """, id=vuln_id, user_id=user_id, project_id=project_id, - props=vuln_props + props=vuln_props, + **self._recon_job_params(), ) record = result.single() @@ -353,7 +375,15 @@ def _is_confirmed_graphql(info: dict) -> bool: project_id: $project_id }) ON CREATE SET bu.source = 'graphql_scan', - bu.updated_at = datetime() + bu.updated_at = datetime(), + bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at), + bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id), + bu.last_seen = $recon_job_started_at, + bu.last_seen_job_id = $recon_job_id + ON MATCH SET bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at), + bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id), + bu.last_seen = $recon_job_started_at, + bu.last_seen_job_id = $recon_job_id MERGE (e:Endpoint { path: $path, method: 'POST', @@ -363,7 +393,15 @@ def _is_confirmed_graphql(info: dict) -> bool: }) ON CREATE SET e.source = 'graphql_scan', e.is_graphql = true, - e.created_at = datetime() + e.created_at = datetime(), + e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id + ON MATCH SET e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id MERGE (bu)-[:HAS_ENDPOINT]->(e) MERGE (e)-[:HAS_VULNERABILITY]->(v) """, @@ -371,7 +409,8 @@ def _is_confirmed_graphql(info: dict) -> bool: path=path, baseurl=baseurl, user_id=user_id, - project_id=project_id + project_id=project_id, + **self._recon_job_params(), ) stats["relationships_created"] += 1 @@ -403,13 +442,18 @@ def _is_confirmed_graphql(info: dict) -> bool: user_id: $user_id, project_id: $project_id }) - SET v += $props + SET v += $props, + v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id), + v.last_seen = $recon_job_started_at, + v.last_seen_job_id = $recon_job_id RETURN v.id as id """, id=sensitive_vuln_id, user_id=user_id, project_id=project_id, - props=sensitive_props + props=sensitive_props, + **self._recon_job_params(), ) if result.single(): @@ -429,7 +473,15 @@ def _is_confirmed_graphql(info: dict) -> bool: project_id: $project_id }) ON CREATE SET bu.source = 'graphql_scan', - bu.updated_at = datetime() + bu.updated_at = datetime(), + bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at), + bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id), + bu.last_seen = $recon_job_started_at, + bu.last_seen_job_id = $recon_job_id + ON MATCH SET bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at), + bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id), + bu.last_seen = $recon_job_started_at, + bu.last_seen_job_id = $recon_job_id MERGE (e:Endpoint { path: $path, method: 'POST', @@ -439,7 +491,15 @@ def _is_confirmed_graphql(info: dict) -> bool: }) ON CREATE SET e.source = 'graphql_scan', e.is_graphql = true, - e.created_at = datetime() + e.created_at = datetime(), + e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id + ON MATCH SET e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id MERGE (bu)-[:HAS_ENDPOINT]->(e) MERGE (e)-[:HAS_VULNERABILITY]->(v) """, @@ -447,7 +507,8 @@ def _is_confirmed_graphql(info: dict) -> bool: path=path, baseurl=baseurl, user_id=user_id, - project_id=project_id + project_id=project_id, + **self._recon_job_params(), ) stats["relationships_created"] += 1 @@ -464,4 +525,4 @@ def _is_confirmed_graphql(info: dict) -> bool: for error in stats["errors"][:5]: # Show first 5 errors print(f"[!][graph-db] {error}") - return stats \ No newline at end of file + return stats diff --git a/graph_db/mixins/osint_mixin.py b/graph_db/mixins/osint_mixin.py index 3cf9e930..5d691d0d 100644 --- a/graph_db/mixins/osint_mixin.py +++ b/graph_db/mixins/osint_mixin.py @@ -75,9 +75,15 @@ def update_graph_from_shodan(self, recon_data: dict, user_id: str, project_id: s session.run( """ MERGE (i:IP {address: $address, user_id: $user_id, project_id: $project_id}) - SET i += $props, i.updated_at = datetime() + SET i += $props, + i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, - address=ip, user_id=user_id, project_id=project_id, props=props + address=ip, user_id=user_id, project_id=project_id, props=props, + **self._recon_job_params(), ) stats["ips_enriched"] += 1 @@ -93,13 +99,26 @@ def update_graph_from_shodan(self, recon_data: dict, user_id: str, project_id: s """ MERGE (p:Port {number: $port, protocol: $protocol, ip_address: $ip, user_id: $user_id, project_id: $project_id}) - ON CREATE SET p.state = 'open', p.source = 'shodan', p.updated_at = datetime() - ON MATCH SET p.updated_at = datetime() + ON CREATE SET p.state = 'open', p.source = 'shodan', p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id + ON MATCH SET p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (i)-[:HAS_PORT]->(p) """, port=port_num, protocol=protocol, ip=ip, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["ports_created"] += 1 stats["relationships_created"] += 1 @@ -118,15 +137,24 @@ def update_graph_from_shodan(self, recon_data: dict, user_id: str, project_id: s """ MERGE (svc:Service {name: $name, port_number: $port, ip_address: $ip, user_id: $user_id, project_id: $project_id}) - ON CREATE SET svc += $props, svc.updated_at = datetime() - ON MATCH SET svc.updated_at = datetime() + ON CREATE SET svc += $props, svc.updated_at = datetime(), + svc.first_seen = coalesce(svc.first_seen, $recon_job_started_at), + svc.first_seen_job_id = coalesce(svc.first_seen_job_id, $recon_job_id), + svc.last_seen = $recon_job_started_at, + svc.last_seen_job_id = $recon_job_id + ON MATCH SET svc.updated_at = datetime(), + svc.first_seen = coalesce(svc.first_seen, $recon_job_started_at), + svc.first_seen_job_id = coalesce(svc.first_seen_job_id, $recon_job_id), + svc.last_seen = $recon_job_started_at, + svc.last_seen_job_id = $recon_job_id WITH svc MATCH (p:Port {number: $port, protocol: $protocol, ip_address: $ip, user_id: $user_id, project_id: $project_id}) MERGE (p)-[:RUNS_SERVICE]->(svc) """, name=product, port=port_num, protocol=protocol, ip=ip, - user_id=user_id, project_id=project_id, props=svc_props + user_id=user_id, project_id=project_id, props=svc_props, + **self._recon_job_params(), ) stats["services_created"] += 1 stats["relationships_created"] += 1 @@ -142,13 +170,26 @@ def update_graph_from_shodan(self, recon_data: dict, user_id: str, project_id: s """ MERGE (p:Port {number: $port, protocol: $protocol, ip_address: $ip, user_id: $user_id, project_id: $project_id}) - ON CREATE SET p.state = 'open', p.source = 'shodan', p.updated_at = datetime() - ON MATCH SET p.updated_at = datetime() + ON CREATE SET p.state = 'open', p.source = 'shodan', p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id + ON MATCH SET p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (i)-[:HAS_PORT]->(p) """, port=port_num, protocol="tcp", ip=ip, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["ports_created"] += 1 stats["relationships_created"] += 1 @@ -172,11 +213,24 @@ def update_graph_from_shodan(self, recon_data: dict, user_id: str, project_id: s """ MERGE (s:Subdomain {name: $name, user_id: $user_id, project_id: $project_id}) ON CREATE SET s.source = 'shodan_rdns', s.status = 'resolved', - s.discovered_at = datetime(), s.updated_at = datetime() + s.discovered_at = datetime(), s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id + ON MATCH SET s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (s)-[:RESOLVES_TO {record_type: 'A', timestamp: datetime()}]->(i) """, - name=hostname, ip=ip, user_id=user_id, project_id=project_id + name=hostname, ip=ip, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["subdomains_created"] += 1 stats["relationships_created"] += 1 @@ -202,13 +256,18 @@ def update_graph_from_shodan(self, recon_data: dict, user_id: str, project_id: s ON CREATE SET ed.first_seen_at = datetime() SET ed.sources = coalesce(ed.sources, []) + CASE WHEN NOT 'shodan_rdns' IN coalesce(ed.sources, []) THEN ['shodan_rdns'] ELSE [] END, ed.ips_seen = coalesce(ed.ips_seen, []) + CASE WHEN NOT $ip IN coalesce(ed.ips_seen, []) THEN [$ip] ELSE [] END, - ed.updated_at = datetime() + ed.updated_at = datetime(), + ed.first_seen = coalesce(ed.first_seen, $recon_job_started_at), + ed.first_seen_job_id = coalesce(ed.first_seen_job_id, $recon_job_id), + ed.last_seen = $recon_job_started_at, + ed.last_seen_job_id = $recon_job_id WITH ed MATCH (d:Domain {name: $domain, user_id: $user_id, project_id: $project_id}) MERGE (ed)-[:DISCOVERED_BY]->(d) """, ed_domain=hostname, ip=ip, domain=domain, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["relationships_created"] += 1 @@ -231,9 +290,18 @@ def update_graph_from_shodan(self, recon_data: dict, user_id: str, project_id: s """ MERGE (s:Subdomain {name: $name, user_id: $user_id, project_id: $project_id}) ON CREATE SET s.source = 'shodan_dns', s.status = 'resolved', - s.discovered_at = datetime(), s.updated_at = datetime() + s.discovered_at = datetime(), s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id + ON MATCH SET s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id """, - name=fqdn, user_id=user_id, project_id=project_id + name=fqdn, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["subdomains_created"] += 1 @@ -256,13 +324,18 @@ def update_graph_from_shodan(self, recon_data: dict, user_id: str, project_id: s MERGE (ed:ExternalDomain {domain: $ed_domain, user_id: $user_id, project_id: $project_id}) ON CREATE SET ed.first_seen_at = datetime() SET ed.sources = coalesce(ed.sources, []) + CASE WHEN NOT 'shodan_dns' IN coalesce(ed.sources, []) THEN ['shodan_dns'] ELSE [] END, - ed.updated_at = datetime() + ed.updated_at = datetime(), + ed.first_seen = coalesce(ed.first_seen, $recon_job_started_at), + ed.first_seen_job_id = coalesce(ed.first_seen_job_id, $recon_job_id), + ed.last_seen = $recon_job_started_at, + ed.last_seen_job_id = $recon_job_id WITH ed MATCH (d:Domain {name: $domain, user_id: $user_id, project_id: $project_id}) MERGE (ed)-[:DISCOVERED_BY]->(d) """, ed_domain=fqdn, domain=domain, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) except Exception as e: @@ -280,10 +353,20 @@ def update_graph_from_shodan(self, recon_data: dict, user_id: str, project_id: s """ MERGE (dns:DNSRecord {type: $type, value: $value, subdomain: $subdomain, user_id: $user_id, project_id: $project_id}) - ON CREATE SET dns.source = 'shodan', dns.updated_at = datetime() + ON CREATE SET dns.source = 'shodan', + dns.updated_at = datetime(), + dns.first_seen = coalesce(dns.first_seen, $recon_job_started_at), + dns.first_seen_job_id = coalesce(dns.first_seen_job_id, $recon_job_id), + dns.last_seen = $recon_job_started_at, + dns.last_seen_job_id = $recon_job_id + ON MATCH SET dns.first_seen = coalesce(dns.first_seen, $recon_job_started_at), + dns.first_seen_job_id = coalesce(dns.first_seen_job_id, $recon_job_id), + dns.last_seen = $recon_job_started_at, + dns.last_seen_job_id = $recon_job_id """, type=rec_type, value=rec_value, subdomain=fqdn, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["dns_records_created"] += 1 @@ -293,10 +376,15 @@ def update_graph_from_shodan(self, recon_data: dict, user_id: str, project_id: s """ MATCH (s:Subdomain {name: $subdomain, user_id: $user_id, project_id: $project_id}) MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (s)-[:RESOLVES_TO {record_type: $type, timestamp: datetime()}]->(i) """, subdomain=fqdn, ip=rec_value, type=rec_type, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["relationships_created"] += 1 @@ -317,10 +405,19 @@ def update_graph_from_shodan(self, recon_data: dict, user_id: str, project_id: s MERGE (v:Vulnerability {id: $vuln_id}) ON CREATE SET v.source = $source, v.name = $cve_id, v.cves = [$cve_id], v.user_id = $user_id, - v.project_id = $project_id, v.updated_at = datetime() + v.project_id = $project_id, v.updated_at = datetime(), + v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id), + v.last_seen = $recon_job_started_at, + v.last_seen_job_id = $recon_job_id + ON MATCH SET v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id), + v.last_seen = $recon_job_started_at, + v.last_seen_job_id = $recon_job_id """, vuln_id=vuln_id, cve_id=cve_id, source=cve_source, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["vulnerabilities_created"] += 1 @@ -423,11 +520,20 @@ def update_graph_from_urlscan_discovery(self, recon_data: dict, user_id: str, pr MERGE (d:Domain {name: $domain, user_id: $uid, project_id: $pid}) MERGE (s:Subdomain {name: $subdomain, user_id: $uid, project_id: $pid}) ON CREATE SET s.discovered_by = 'urlscan', s.status = 'resolved', - s.updated_at = datetime() + s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id + ON MATCH SET s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id MERGE (d)-[:HAS_SUBDOMAIN]->(s) """, domain=domain, subdomain=subdomain, - uid=user_id, pid=project_id + uid=user_id, pid=project_id, + **self._recon_job_params(), ) stats["subdomains_created"] += 1 stats["relationships_created"] += 1 @@ -437,13 +543,18 @@ def update_graph_from_urlscan_discovery(self, recon_data: dict, user_id: str, pr MERGE (ed:ExternalDomain {domain: $ed_domain, user_id: $uid, project_id: $pid}) ON CREATE SET ed.first_seen_at = datetime() SET ed.sources = coalesce(ed.sources, []) + CASE WHEN NOT 'urlscan' IN coalesce(ed.sources, []) THEN ['urlscan'] ELSE [] END, - ed.updated_at = datetime() + ed.updated_at = datetime(), + ed.first_seen = coalesce(ed.first_seen, $recon_job_started_at), + ed.first_seen_job_id = coalesce(ed.first_seen_job_id, $recon_job_id), + ed.last_seen = $recon_job_started_at, + ed.last_seen_job_id = $recon_job_id WITH ed MATCH (d:Domain {name: $domain, user_id: $uid, project_id: $pid}) MERGE (ed)-[:DISCOVERED_BY]->(d) """, ed_domain=subdomain, domain=domain, - uid=user_id, pid=project_id + uid=user_id, pid=project_id, + **self._recon_job_params(), ) except Exception as e: stats["errors"].append(f"Subdomain {subdomain}: {e}") @@ -462,9 +573,15 @@ def update_graph_from_urlscan_discovery(self, recon_data: dict, user_id: str, pr session.run( """ MERGE (i:IP {address: $ip, user_id: $uid, project_id: $pid}) - SET i += $props, i.updated_at = datetime() + SET i += $props, + i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, - ip=ip, uid=user_id, pid=project_id, props=props + ip=ip, uid=user_id, pid=project_id, props=props, + **self._recon_job_params(), ) stats["ips_enriched"] += 1 except Exception as e: @@ -480,10 +597,15 @@ def update_graph_from_urlscan_discovery(self, recon_data: dict, user_id: str, pr """ MATCH (s:Subdomain {name: $subdomain, user_id: $uid, project_id: $pid}) MERGE (i:IP {address: $ip, user_id: $uid, project_id: $pid}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (s)-[:RESOLVES_TO]->(i) """, subdomain=subdomain, ip=ip, - uid=user_id, pid=project_id + uid=user_id, pid=project_id, + **self._recon_job_params(), ) stats["relationships_created"] += 1 except Exception as e: @@ -574,10 +696,16 @@ def update_graph_from_urlscan_enrichment(self, recon_data: dict, user_id: str, p result = session.run( """ MATCH (bu:BaseURL {url: $base_url, user_id: $uid, project_id: $pid}) - SET bu += $props, bu.updated_at = datetime() + SET bu += $props, + bu.updated_at = datetime(), + bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at), + bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id), + bu.last_seen = $recon_job_started_at, + bu.last_seen_job_id = $recon_job_id RETURN bu.url AS url """, - base_url=base_url, uid=user_id, pid=project_id, props=props + base_url=base_url, uid=user_id, pid=project_id, props=props, + **self._recon_job_params(), ) if result.single(): stats["baseurls_enriched"] += 1 @@ -605,12 +733,21 @@ def update_graph_from_urlscan_enrichment(self, recon_data: dict, user_id: str, p MERGE (e:Endpoint {path: $path, method: 'GET', baseurl: $base_url, user_id: $uid, project_id: $pid}) ON CREATE SET e.source = 'urlscan', e.full_url = $full_url, - e.has_parameters = $has_params, e.updated_at = datetime() + e.has_parameters = $has_params, e.updated_at = datetime(), + e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id + ON MATCH SET e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id MERGE (bu)-[:HAS_ENDPOINT]->(e) RETURN e.path AS path """, base_url=base_url, path=path, full_url=full_url, - has_params=has_params, uid=user_id, pid=project_id + has_params=has_params, uid=user_id, pid=project_id, + **self._recon_job_params(), ) record = result.single() if record: @@ -629,12 +766,21 @@ def update_graph_from_urlscan_enrichment(self, recon_data: dict, user_id: str, p endpoint_path: $path, baseurl: $base_url, user_id: $uid, project_id: $pid}) ON CREATE SET p.source = 'urlscan', p.sample_value = $sample_val, - p.is_injectable = false, p.updated_at = datetime() + p.is_injectable = false, p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id + ON MATCH SET p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id MERGE (e)-[:HAS_PARAMETER]->(p) """, path=path, base_url=base_url, param_name=param_name, sample_val=sample_val[:500], - uid=user_id, pid=project_id + uid=user_id, pid=project_id, + **self._recon_job_params(), ) stats["parameters_created"] += 1 stats["relationships_created"] += 1 @@ -691,7 +837,11 @@ def update_graph_from_external_domains(self, recon_data, user_id, project_id): ed.ips_seen = $ips, ed.countries_seen = $countries, ed.times_seen = $times_seen, - ed.updated_at = datetime() + ed.updated_at = datetime(), + ed.first_seen = coalesce(ed.first_seen, $recon_job_started_at), + ed.first_seen_job_id = coalesce(ed.first_seen_job_id, $recon_job_id), + ed.last_seen = $recon_job_started_at, + ed.last_seen_job_id = $recon_job_id RETURN ed.first_seen_at = ed.updated_at AS is_new """, ed_domain=ed_domain, uid=user_id, pid=project_id, sources=ed.get("sources", []), @@ -703,6 +853,7 @@ def update_graph_from_external_domains(self, recon_data, user_id, project_id): ips=ed.get("ips_seen", []), countries=ed.get("countries_seen", []), times_seen=ed.get("times_seen", 0), + **self._recon_job_params(), ) record = result.single() if record and record["is_new"]: @@ -787,9 +938,15 @@ def update_graph_from_censys(self, recon_data: dict, user_id: str, project_id: s session.run( """ MERGE (i:IP {address: $address, user_id: $user_id, project_id: $project_id}) - SET i += $props, i.updated_at = datetime() + SET i += $props, + i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, address=ip, user_id=user_id, project_id=project_id, props=ip_props, + **self._recon_job_params(), ) stats["ips_enriched"] += 1 @@ -806,12 +963,21 @@ def update_graph_from_censys(self, recon_data: dict, user_id: str, project_id: s MERGE (p:Port {number: $port, protocol: $protocol, ip_address: $ip, user_id: $user_id, project_id: $project_id}) ON CREATE SET p.state = 'open', p.updated_at = datetime() - SET p.source = 'censys', p.updated_at = datetime() + SET p.source = 'censys', p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (i)-[:HAS_PORT]->(p) """, port=int(port_num), protocol=protocol, ip=ip, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["ports_merged"] += 1 stats["relationships_created"] += 1 @@ -847,7 +1013,11 @@ def update_graph_from_censys(self, recon_data: dict, user_id: str, project_id: s MERGE (svc:Service {name: $name, port_number: $port, ip_address: $ip, user_id: $user_id, project_id: $project_id}) ON CREATE SET svc.updated_at = datetime() - SET svc += $props, svc.updated_at = datetime() + SET svc += $props, svc.updated_at = datetime(), + svc.first_seen = coalesce(svc.first_seen, $recon_job_started_at), + svc.first_seen_job_id = coalesce(svc.first_seen_job_id, $recon_job_id), + svc.last_seen = $recon_job_started_at, + svc.last_seen_job_id = $recon_job_id WITH svc MATCH (p:Port {number: $port, protocol: $protocol, ip_address: $ip, user_id: $user_id, project_id: $project_id}) @@ -855,6 +1025,7 @@ def update_graph_from_censys(self, recon_data: dict, user_id: str, project_id: s """, name=sname, port=int(port_num), protocol=protocol, ip=ip, user_id=user_id, project_id=project_id, props=svc_props, + **self._recon_job_params(), ) stats["services_merged"] += 1 stats["relationships_created"] += 1 @@ -880,13 +1051,19 @@ def update_graph_from_censys(self, recon_data: dict, user_id: str, project_id: s MERGE (c:Certificate {subject_cn: $subject_cn, user_id: $user_id, project_id: $project_id}) - SET c += $props, c.updated_at = datetime() + SET c += $props, + c.updated_at = datetime(), + c.first_seen = coalesce(c.first_seen, $recon_job_started_at), + c.first_seen_job_id = coalesce(c.first_seen_job_id, $recon_job_id), + c.last_seen = $recon_job_started_at, + c.last_seen_job_id = $recon_job_id WITH c MATCH (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) MERGE (i)-[:HAS_CERTIFICATE]->(c) """, subject_cn=subject_cn, user_id=user_id, project_id=project_id, props=cert_props, ip=ip, + **self._recon_job_params(), ) stats["certificates_merged"] += 1 stats["relationships_created"] += 1 @@ -905,11 +1082,20 @@ def update_graph_from_censys(self, recon_data: dict, user_id: str, project_id: s MERGE (s:Subdomain {name: $name, user_id: $user_id, project_id: $project_id}) ON CREATE SET s.discovered_at = datetime(), s.updated_at = datetime() SET s.source = 'censys_rdns', s.status = 'unverified', - s.updated_at = datetime() + s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (s)-[:RESOLVES_TO {record_type: 'A', timestamp: datetime()}]->(i) """, name=hostname, ip=ip, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) session.run( """ @@ -977,9 +1163,15 @@ def update_graph_from_fofa(self, recon_data: dict, user_id: str, project_id: str session.run( """ MERGE (i:IP {address: $address, user_id: $user_id, project_id: $project_id}) - SET i += $props, i.updated_at = datetime() + SET i += $props, + i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, address=ip, user_id=user_id, project_id=project_id, props=ip_props, + **self._recon_job_params(), ) stats["ips_enriched"] += 1 @@ -996,11 +1188,20 @@ def update_graph_from_fofa(self, recon_data: dict, user_id: str, project_id: str MERGE (p:Port {number: $port, protocol: 'tcp', ip_address: $ip, user_id: $user_id, project_id: $project_id}) ON CREATE SET p.state = 'open', p.updated_at = datetime() - SET p.source = 'fofa', p.updated_at = datetime() + SET p.source = 'fofa', p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (i)-[:HAS_PORT]->(p) """, port=pnum, ip=ip, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["ports_merged"] += 1 stats["relationships_created"] += 1 @@ -1022,7 +1223,11 @@ def update_graph_from_fofa(self, recon_data: dict, user_id: str, project_id: str svc.version = CASE WHEN $version <> '' THEN $version ELSE svc.version END, svc.app_protocol = CASE WHEN $app_proto <> '' THEN $app_proto ELSE svc.app_protocol END, svc.jarm = CASE WHEN $jarm <> '' THEN $jarm ELSE svc.jarm END, - svc.tls_version = CASE WHEN $tls_ver <> '' THEN $tls_ver ELSE svc.tls_version END + svc.tls_version = CASE WHEN $tls_ver <> '' THEN $tls_ver ELSE svc.tls_version END, + svc.first_seen = coalesce(svc.first_seen, $recon_job_started_at), + svc.first_seen_job_id = coalesce(svc.first_seen_job_id, $recon_job_id), + svc.last_seen = $recon_job_started_at, + svc.last_seen_job_id = $recon_job_id WITH svc MATCH (p:Port {number: $port, protocol: 'tcp', ip_address: $ip, user_id: $user_id, project_id: $project_id}) @@ -1036,6 +1241,7 @@ def update_graph_from_fofa(self, recon_data: dict, user_id: str, project_id: str jarm=row.get("jarm") or "", tls_ver=row.get("tls_version") or "", user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["services_merged"] += 1 stats["relationships_created"] += 1 @@ -1046,13 +1252,25 @@ def update_graph_from_fofa(self, recon_data: dict, user_id: str, project_id: str session.run( """ MERGE (c:Certificate {subject_cn: $cn, user_id: $user_id, project_id: $project_id}) - ON CREATE SET c.source = 'fofa', c.updated_at = datetime() + ON CREATE SET c.source = 'fofa', c.updated_at = datetime(), + c.first_seen = coalesce(c.first_seen, $recon_job_started_at), + c.first_seen_job_id = coalesce(c.first_seen_job_id, $recon_job_id), + c.last_seen = $recon_job_started_at, + c.last_seen_job_id = $recon_job_id SET c.issuer = CASE WHEN $issuer_cn <> '' THEN $issuer_cn ELSE c.issuer END, c.subject_org = CASE WHEN $subject_org <> '' THEN $subject_org ELSE c.subject_org END, c.tls_version = CASE WHEN $tls_ver <> '' THEN $tls_ver ELSE c.tls_version END, c.is_valid = CASE WHEN $cert_valid <> '' THEN ($cert_valid = 'true') ELSE c.is_valid END, - c.updated_at = datetime() + c.updated_at = datetime(), + c.first_seen = coalesce(c.first_seen, $recon_job_started_at), + c.first_seen_job_id = coalesce(c.first_seen_job_id, $recon_job_id), + c.last_seen = $recon_job_started_at, + c.last_seen_job_id = $recon_job_id MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (i)-[:HAS_CERTIFICATE]->(c) """, cn=cert_cn, @@ -1061,6 +1279,7 @@ def update_graph_from_fofa(self, recon_data: dict, user_id: str, project_id: str tls_ver=row.get("tls_version") or "", cert_valid=str(row.get("certs_valid") or "").lower(), ip=ip, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["certificates_merged"] += 1 stats["relationships_created"] += 1 @@ -1076,11 +1295,20 @@ def update_graph_from_fofa(self, recon_data: dict, user_id: str, project_id: str """ MERGE (s:Subdomain {name: $name, user_id: $user_id, project_id: $project_id}) ON CREATE SET s.status = 'unverified', s.discovered_at = datetime(), s.updated_at = datetime() - SET s.source = 'fofa', s.updated_at = datetime() + SET s.source = 'fofa', s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (s)-[:RESOLVES_TO {record_type: 'A', timestamp: datetime()}]->(i) """, name=host, ip=ip, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["subdomains_merged"] += 1 stats["relationships_created"] += 1 @@ -1147,7 +1375,11 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) i.country_name = CASE WHEN i.country_name IS NULL OR i.country_name = '' THEN $country_name ELSE i.country_name END, i.city = CASE WHEN i.city IS NULL OR i.city = '' THEN $city ELSE i.city END, i.asn = CASE WHEN i.asn IS NULL OR i.asn = '' THEN $asn ELSE i.asn END, - i.updated_at = datetime() + i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, address=ip, user_id=user_id, project_id=project_id, pulse=rep.get("pulse_count"), @@ -1160,6 +1392,7 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) country_name=geo.get("country_name") or "", city=geo.get("city") or "", asn=geo.get("asn") or "", + **self._recon_job_params(), ) stats["ips_enriched"] += 1 @@ -1189,15 +1422,24 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) """ MERGE (s:Subdomain {name: $name, user_id: $user_id, project_id: $project_id}) ON CREATE SET s.discovered_at = datetime(), s.updated_at = datetime() - SET s.source = 'otx_passive_dns', s.status = 'unverified', s.updated_at = datetime() + SET s.source = 'otx_passive_dns', s.status = 'unverified', s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id WITH s MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (s)-[r:RESOLVES_TO {record_type: $record_type}]->(i) ON CREATE SET r.first_seen = $first_seen, r.last_seen = $last_seen, r.timestamp = datetime() SET r.last_seen = CASE WHEN $last_seen <> '' THEN $last_seen ELSE r.last_seen END """, name=hostname, ip=ip, user_id=user_id, project_id=project_id, record_type=record_type, first_seen=first_seen, last_seen=last_seen, + **self._recon_job_params(), ) stats["subdomains_merged"] += 1 stats["relationships_created"] += 1 @@ -1223,13 +1465,18 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) ON CREATE SET ed.first_seen_at = datetime() SET ed.sources = coalesce(ed.sources, []) + CASE WHEN NOT 'otx_passive_dns' IN coalesce(ed.sources, []) THEN ['otx_passive_dns'] ELSE [] END, ed.ips_seen = coalesce(ed.ips_seen, []) + CASE WHEN NOT $ip IN coalesce(ed.ips_seen, []) THEN [$ip] ELSE [] END, - ed.updated_at = datetime() + ed.updated_at = datetime(), + ed.first_seen = coalesce(ed.first_seen, $recon_job_started_at), + ed.first_seen_job_id = coalesce(ed.first_seen_job_id, $recon_job_id), + ed.last_seen = $recon_job_started_at, + ed.last_seen_job_id = $recon_job_id WITH ed MATCH (d:Domain {name: $domain, user_id: $user_id, project_id: $project_id}) MERGE (d)-[:HAS_EXTERNAL_DOMAIN]->(ed) """, ed_domain=hostname, ip=ip, domain=domain, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["external_domains_merged"] += 1 stats["relationships_created"] += 1 @@ -1247,14 +1494,23 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) session.run( """ MERGE (m:Malware {hash: $hash, user_id: $user_id, project_id: $project_id}) - ON CREATE SET m.first_seen = datetime() + ON CREATE SET m.first_seen = coalesce(m.first_seen, $recon_job_started_at), + m.first_seen_job_id = coalesce(m.first_seen_job_id, $recon_job_id) SET m.hash_type = $hash_type, m.file_type = $file_type, m.file_name = $file_name, m.source = 'otx', - m.updated_at = datetime() + m.updated_at = datetime(), + m.first_seen = coalesce(m.first_seen, $recon_job_started_at), + m.first_seen_job_id = coalesce(m.first_seen_job_id, $recon_job_id), + m.last_seen = $recon_job_started_at, + m.last_seen_job_id = $recon_job_id WITH m MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (i)-[:ASSOCIATED_WITH_MALWARE]->(m) """, hash=h, user_id=user_id, project_id=project_id, @@ -1262,6 +1518,7 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) file_type=sample.get("file_type") or "", file_name=sample.get("file_name") or "", ip=ip, + **self._recon_job_params(), ) stats["malware_merged"] += 1 stats["relationships_created"] += 1 @@ -1289,9 +1546,17 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) tp.author_name = $author_name, tp.targeted_countries = $targeted_countries, tp.modified = $modified, - tp.updated_at = datetime() + tp.updated_at = datetime(), + tp.first_seen = coalesce(tp.first_seen, $recon_job_started_at), + tp.first_seen_job_id = coalesce(tp.first_seen_job_id, $recon_job_id), + tp.last_seen = $recon_job_started_at, + tp.last_seen_job_id = $recon_job_id WITH tp MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (i)-[:APPEARS_IN_PULSE]->(tp) """, pulse_id=pulse_id, user_id=user_id, project_id=project_id, @@ -1305,6 +1570,7 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) targeted_countries=pulse.get("targeted_countries") or [], modified=pulse.get("modified") or "", ip=ip, + **self._recon_job_params(), ) stats["threat_pulses_merged"] += 1 stats["relationships_created"] += 1 @@ -1363,12 +1629,17 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) session.run( """ MERGE (m:Malware {hash: $hash, user_id: $user_id, project_id: $project_id}) - ON CREATE SET m.first_seen = datetime() + ON CREATE SET m.first_seen = coalesce(m.first_seen, $recon_job_started_at), + m.first_seen_job_id = coalesce(m.first_seen_job_id, $recon_job_id) SET m.hash_type = $hash_type, m.file_type = $file_type, m.file_name = $file_name, m.source = 'otx', - m.updated_at = datetime() + m.updated_at = datetime(), + m.first_seen = coalesce(m.first_seen, $recon_job_started_at), + m.first_seen_job_id = coalesce(m.first_seen_job_id, $recon_job_id), + m.last_seen = $recon_job_started_at, + m.last_seen_job_id = $recon_job_id WITH m MATCH (d:Domain {name: $dom_name, user_id: $user_id, project_id: $project_id}) MERGE (d)-[:ASSOCIATED_WITH_MALWARE]->(m) @@ -1378,6 +1649,7 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) file_type=sample.get("file_type") or "", file_name=sample.get("file_name") or "", dom_name=dom_name, + **self._recon_job_params(), ) stats["malware_merged"] += 1 stats["relationships_created"] += 1 @@ -1405,7 +1677,11 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) tp.author_name = $author_name, tp.targeted_countries = $targeted_countries, tp.modified = $modified, - tp.updated_at = datetime() + tp.updated_at = datetime(), + tp.first_seen = coalesce(tp.first_seen, $recon_job_started_at), + tp.first_seen_job_id = coalesce(tp.first_seen_job_id, $recon_job_id), + tp.last_seen = $recon_job_started_at, + tp.last_seen_job_id = $recon_job_id WITH tp MATCH (d:Domain {name: $dom_name, user_id: $user_id, project_id: $project_id}) MERGE (d)-[:APPEARS_IN_PULSE]->(tp) @@ -1421,6 +1697,7 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) targeted_countries=pulse.get("targeted_countries") or [], modified=pulse.get("modified") or "", dom_name=dom_name, + **self._recon_job_params(), ) stats["threat_pulses_merged"] += 1 stats["relationships_created"] += 1 @@ -1439,7 +1716,11 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) """ MERGE (i:IP {address: $address, user_id: $user_id, project_id: $project_id}) ON CREATE SET i.created_at = datetime() - SET i.updated_at = datetime() + SET i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id WITH i MATCH (d:Domain {name: $dom_name, user_id: $user_id, project_id: $project_id}) MERGE (d)-[r:HISTORICALLY_RESOLVED_TO]->(i) @@ -1450,6 +1731,7 @@ def update_graph_from_otx(self, recon_data: dict, user_id: str, project_id: str) first_seen=hist.get("first") or "", last_seen=hist.get("last") or "", record_type=hist.get("record_type") or "A", + **self._recon_job_params(), ) stats["relationships_created"] += 1 except Exception as e2: @@ -1505,9 +1787,15 @@ def update_graph_from_netlas(self, recon_data: dict, user_id: str, project_id: s session.run( """ MERGE (i:IP {address: $address, user_id: $user_id, project_id: $project_id}) - SET i += $props, i.updated_at = datetime() + SET i += $props, + i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, address=ip, user_id=user_id, project_id=project_id, props=ip_props, + **self._recon_job_params(), ) stats["ips_enriched"] += 1 @@ -1526,11 +1814,20 @@ def update_graph_from_netlas(self, recon_data: dict, user_id: str, project_id: s MERGE (p:Port {number: $port, protocol: $protocol, ip_address: $ip, user_id: $user_id, project_id: $project_id}) ON CREATE SET p.state = 'open', p.updated_at = datetime() - SET p.source = 'netlas', p.updated_at = datetime() + SET p.source = 'netlas', p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (i)-[:HAS_PORT]->(p) """, port=pnum, protocol=protocol, ip=ip, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["ports_merged"] += 1 stats["relationships_created"] += 1 @@ -1556,7 +1853,11 @@ def update_graph_from_netlas(self, recon_data: dict, user_id: str, project_id: s MERGE (svc:Service {name: $name, port_number: $port, ip_address: $ip, user_id: $user_id, project_id: $project_id}) ON CREATE SET svc.updated_at = datetime() - SET svc += $props, svc.updated_at = datetime() + SET svc += $props, svc.updated_at = datetime(), + svc.first_seen = coalesce(svc.first_seen, $recon_job_started_at), + svc.first_seen_job_id = coalesce(svc.first_seen_job_id, $recon_job_id), + svc.last_seen = $recon_job_started_at, + svc.last_seen_job_id = $recon_job_id WITH svc MATCH (p:Port {number: $port, protocol: $protocol, ip_address: $ip, user_id: $user_id, project_id: $project_id}) @@ -1564,6 +1865,7 @@ def update_graph_from_netlas(self, recon_data: dict, user_id: str, project_id: s """, name=prot_name, port=pnum, protocol=protocol, ip=ip, user_id=user_id, project_id=project_id, props=svc_props, + **self._recon_job_params(), ) stats["services_merged"] += 1 stats["relationships_created"] += 1 @@ -1596,7 +1898,11 @@ def update_graph_from_netlas(self, recon_data: dict, user_id: str, project_id: s user_id: $user_id, project_id: $project_id}) ON CREATE SET v.created_at = datetime() - SET v += $props, v.updated_at = datetime() + SET v += $props, v.updated_at = datetime(), + v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id), + v.last_seen = $recon_job_started_at, + v.last_seen_job_id = $recon_job_id WITH v MATCH (svc:Service {name: $svc_name, port_number: $port, ip_address: $ip, @@ -1606,6 +1912,7 @@ def update_graph_from_netlas(self, recon_data: dict, user_id: str, project_id: s """, cve_id=cve_id, user_id=user_id, project_id=project_id, props=vuln_props, svc_name=prot_name, port=pnum, ip=ip, + **self._recon_job_params(), ) stats["vulnerabilities_merged"] += 1 stats["relationships_created"] += 1 @@ -1703,7 +2010,11 @@ def update_graph_from_virustotal(self, recon_data: dict, user_id: str, project_i i.asn = CASE WHEN i.asn IS NOT NULL THEN i.asn ELSE $asn END, i.as_owner = CASE WHEN i.as_owner IS NOT NULL THEN i.as_owner ELSE $as_owner END, i.country = CASE WHEN i.country IS NOT NULL THEN i.country ELSE $country END, - i.updated_at = datetime() + i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, address=ip, user_id=user_id, project_id=project_id, reputation=rep.get("reputation"), @@ -1722,6 +2033,7 @@ def update_graph_from_virustotal(self, recon_data: dict, user_id: str, project_i asn=rep.get("asn"), as_owner=rep.get("as_owner"), country=rep.get("country"), + **self._recon_job_params(), ) stats["ips_enriched"] += 1 except Exception as e: @@ -1772,9 +2084,15 @@ def update_graph_from_zoomeye(self, recon_data: dict, user_id: str, project_id: session.run( """ MERGE (i:IP {address: $address, user_id: $user_id, project_id: $project_id}) - SET i += $props, i.updated_at = datetime() + SET i += $props, + i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, address=ip, user_id=user_id, project_id=project_id, props=ip_props, + **self._recon_job_params(), ) stats["ips_enriched"] += 1 @@ -1793,12 +2111,21 @@ def update_graph_from_zoomeye(self, recon_data: dict, user_id: str, project_id: MERGE (p:Port {number: $port, protocol: $protocol, ip_address: $ip, user_id: $user_id, project_id: $project_id}) ON CREATE SET p.state = 'open', p.updated_at = datetime() - SET p.source = 'zoomeye', p.updated_at = datetime() + SET p.source = 'zoomeye', p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (i)-[:HAS_PORT]->(p) """, port=pnum, protocol=protocol, ip=ip, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["ports_merged"] += 1 stats["relationships_created"] += 1 @@ -1825,7 +2152,11 @@ def update_graph_from_zoomeye(self, recon_data: dict, user_id: str, project_id: ip_address: $ip, user_id: $user_id, project_id: $project_id}) ON CREATE SET svc.updated_at = datetime() - SET svc += $props, svc.updated_at = datetime() + SET svc += $props, svc.updated_at = datetime(), + svc.first_seen = coalesce(svc.first_seen, $recon_job_started_at), + svc.first_seen_job_id = coalesce(svc.first_seen_job_id, $recon_job_id), + svc.last_seen = $recon_job_started_at, + svc.last_seen_job_id = $recon_job_id WITH svc MATCH (p:Port {number: $port, protocol: $protocol, ip_address: $ip, @@ -1834,6 +2165,7 @@ def update_graph_from_zoomeye(self, recon_data: dict, user_id: str, project_id: """, name=svc_name, port=pnum, protocol=protocol, ip=ip, props=svc_props, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["services_merged"] += 1 stats["relationships_created"] += 1 @@ -1857,14 +2189,27 @@ def update_graph_from_zoomeye(self, recon_data: dict, user_id: str, project_id: ON CREATE SET s.source = 'zoomeye_rdns', s.status = 'resolved', s.discovered_at = datetime(), - s.updated_at = datetime() + s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id + ON MATCH SET s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (s)-[:RESOLVES_TO {record_type: 'A', timestamp: datetime()}]->(i) """, name=hostname_val, ip=ip, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["subdomains_merged"] += 1 stats["relationships_created"] += 1 @@ -1987,9 +2332,15 @@ def update_graph_from_criminalip(self, recon_data: dict, user_id: str, project_i session.run( """ MERGE (i:IP {address: $address, user_id: $user_id, project_id: $project_id}) - SET i += $props, i.updated_at = datetime() + SET i += $props, + i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, address=ip, user_id=user_id, project_id=project_id, props=ip_props, + **self._recon_job_params(), ) stats["ips_enriched"] += 1 @@ -2009,12 +2360,21 @@ def update_graph_from_criminalip(self, recon_data: dict, user_id: str, project_i MERGE (p:Port {number: $port, protocol: $protocol, ip_address: $ip, user_id: $user_id, project_id: $project_id}) ON CREATE SET p.state = 'open', p.updated_at = datetime() - SET p.source = 'criminalip', p.updated_at = datetime() + SET p.source = 'criminalip', p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (i)-[:HAS_PORT]->(p) """, port=pnum, protocol=proto, ip=ip, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["ports_merged"] += 1 stats["relationships_created"] += 1 @@ -2030,7 +2390,11 @@ def update_graph_from_criminalip(self, recon_data: dict, user_id: str, project_i SET svc.source = 'criminalip', svc.version = $version, svc.banner = $banner, - svc.updated_at = datetime() + svc.updated_at = datetime(), + svc.first_seen = coalesce(svc.first_seen, $recon_job_started_at), + svc.first_seen_job_id = coalesce(svc.first_seen_job_id, $recon_job_id), + svc.last_seen = $recon_job_started_at, + svc.last_seen_job_id = $recon_job_id WITH svc MATCH (p:Port {number: $port, protocol: $protocol, ip_address: $ip, user_id: $user_id, project_id: $project_id}) @@ -2040,6 +2404,7 @@ def update_graph_from_criminalip(self, recon_data: dict, user_id: str, project_i version=pentry.get("app_version"), banner=(pentry.get("banner") or "")[:500] or None, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["services_created"] += 1 stats["relationships_created"] += 1 @@ -2060,10 +2425,19 @@ def update_graph_from_criminalip(self, recon_data: dict, user_id: str, project_i ON CREATE SET v.source = 'criminalip', v.name = $cve_id, v.cves = [$cve_id], v.cvss = $cvss, v.user_id = $user_id, v.project_id = $project_id, - v.updated_at = datetime() + v.updated_at = datetime(), + v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id), + v.last_seen = $recon_job_started_at, + v.last_seen_job_id = $recon_job_id + ON MATCH SET v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id), + v.last_seen = $recon_job_started_at, + v.last_seen_job_id = $recon_job_id """, vuln_id=vuln_id, cve_id=cve_id, cvss=cvss, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["vulnerabilities_created"] += 1 @@ -2144,13 +2518,22 @@ def update_graph_from_uncover(self, recon_data: dict, user_id: str, project_id: """ MERGE (s:Subdomain {name: $name, user_id: $user_id, project_id: $project_id}) ON CREATE SET s.discovered_at = datetime(), s.updated_at = datetime(), - s.source = 'uncover', s.status = 'unverified' + s.source = 'uncover', s.status = 'unverified', + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id SET s.uncover_sources = $sources, s.uncover_total_raw = $total_raw, - s.uncover_total_deduped = $total_deduped + s.uncover_total_deduped = $total_deduped, + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id """, name=hostname, user_id=user_id, project_id=project_id, sources=sources, total_raw=total_raw, total_deduped=total_deduped, + **self._recon_job_params(), ) stats["subdomains_created"] += 1 if domain: @@ -2180,12 +2563,17 @@ def update_graph_from_uncover(self, recon_data: dict, user_id: str, project_id: i.uncover_sources = $sources, i.uncover_source_counts = $source_counts_str, i.uncover_total_raw = $total_raw, - i.uncover_total_deduped = $total_deduped + i.uncover_total_deduped = $total_deduped, + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, address=ip, user_id=user_id, project_id=project_id, sources=sources, source_counts_str=str(source_counts), total_raw=total_raw, total_deduped=total_deduped, + **self._recon_job_params(), ) stats["ips_created"] += 1 @@ -2211,12 +2599,25 @@ def update_graph_from_uncover(self, recon_data: dict, user_id: str, project_id: MERGE (p:Port {number: $port, protocol: 'tcp', ip_address: $ip, user_id: $user_id, project_id: $project_id}) ON CREATE SET p.state = 'open', p.source = 'uncover', - p.updated_at = datetime() + p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id + ON MATCH SET p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id MERGE (i:IP {address: $ip, user_id: $user_id, project_id: $project_id}) + SET i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id MERGE (i)-[:HAS_PORT]->(p) """, port=int(port_num), ip=ip, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["relationships_created"] += 1 except Exception as e: @@ -2230,9 +2631,18 @@ def update_graph_from_uncover(self, recon_data: dict, user_id: str, project_id: """ MERGE (e:Endpoint {url: $url, user_id: $user_id, project_id: $project_id}) ON CREATE SET e.discovered_at = datetime(), e.updated_at = datetime(), - e.source = 'uncover', e.method = 'GET' + e.source = 'uncover', e.method = 'GET', + e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id + ON MATCH SET e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id """, url=url, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["urls_created"] += 1 # Link Endpoint to Domain diff --git a/graph_db/mixins/recon/domain_mixin.py b/graph_db/mixins/recon/domain_mixin.py index a2d24bea..d7c35f7a 100644 --- a/graph_db/mixins/recon/domain_mixin.py +++ b/graph_db/mixins/recon/domain_mixin.py @@ -116,9 +116,14 @@ def update_graph_from_domain_discovery(self, recon_data: dict, user_id: str, pro session.run( """ MERGE (d:Domain {name: $name, user_id: $user_id, project_id: $project_id}) - SET d += $props + SET d += $props, + d.first_seen = coalesce(d.first_seen, $recon_job_started_at), + d.first_seen_job_id = coalesce(d.first_seen_job_id, $recon_job_id), + d.last_seen = $recon_job_started_at, + d.last_seen_job_id = $recon_job_id """, - name=root_domain, user_id=user_id, project_id=project_id, props=domain_props + name=root_domain, user_id=user_id, project_id=project_id, props=domain_props, + **self._recon_job_params(), ) stats["domain_created"] = True print(f"[+][graph-db] Created Domain node: {root_domain}") @@ -151,6 +156,10 @@ def update_graph_from_domain_discovery(self, recon_data: dict, user_id: str, pro s.status = coalesce(s.status, $status), s.discovered_at = coalesce(s.discovered_at, datetime()), s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id, s.ai_service_hint = CASE WHEN $ai_service_hint IS NULL THEN s.ai_service_hint WHEN s.ai_service_hint IS NULL THEN $ai_service_hint @@ -161,6 +170,7 @@ def update_graph_from_domain_discovery(self, recon_data: dict, user_id: str, pro name=subdomain, user_id=user_id, project_id=project_id, has_records=has_records, status=status, ai_service_hint=ai_service_hint, + **self._recon_job_params(), ) stats["subdomains_created"] += 1 if ai_service_hint: @@ -195,10 +205,15 @@ def update_graph_from_domain_discovery(self, recon_data: dict, user_id: str, pro """ MERGE (i:IP {address: $address, user_id: $user_id, project_id: $project_id}) SET i.version = $version, - i.updated_at = datetime() + i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, address=ip_addr, user_id=user_id, project_id=project_id, - version=ip_version + version=ip_version, + **self._recon_job_params(), ) stats["ips_created"] += 1 @@ -232,10 +247,15 @@ def update_graph_from_domain_discovery(self, recon_data: dict, user_id: str, pro MERGE (dns:DNSRecord {type: $type, value: $value, subdomain: $subdomain, user_id: $user_id, project_id: $project_id}) SET dns.user_id = $user_id, dns.project_id = $project_id, - dns.updated_at = datetime() + dns.updated_at = datetime(), + dns.first_seen = coalesce(dns.first_seen, $recon_job_started_at), + dns.first_seen_job_id = coalesce(dns.first_seen_job_id, $recon_job_id), + dns.last_seen = $recon_job_started_at, + dns.last_seen_job_id = $recon_job_id """, type=record_type, value=str(value), subdomain=subdomain, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["dns_records_created"] += 1 @@ -317,9 +337,14 @@ def update_graph_from_ip_recon(self, recon_data: dict, user_id: str, project_id: session.run( """ MERGE (d:Domain {name: $name, user_id: $user_id, project_id: $project_id}) - SET d += $props + SET d += $props, + d.first_seen = coalesce(d.first_seen, $recon_job_started_at), + d.first_seen_job_id = coalesce(d.first_seen_job_id, $recon_job_id), + d.last_seen = $recon_job_started_at, + d.last_seen_job_id = $recon_job_id """, - name=mock_domain, user_id=user_id, project_id=project_id, props=domain_props + name=mock_domain, user_id=user_id, project_id=project_id, props=domain_props, + **self._recon_job_params(), ) stats["domain_created"] = True print(f"[+][graph-db] Created mock Domain node: {mock_domain}") @@ -356,13 +381,22 @@ def update_graph_from_ip_recon(self, recon_data: dict, user_id: str, project_id: session.run( """ MERGE (s:Subdomain {name: $name, user_id: $user_id, project_id: $project_id}) - ON CREATE SET s += $props, s.status = 'resolved' - ON MATCH SET s += $props + ON CREATE SET s += $props, s.status = 'resolved', + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id + ON MATCH SET s += $props, + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id WITH s WHERE s.status IS NULL SET s.status = 'resolved' """, - name=subdomain_name, user_id=user_id, project_id=project_id, props=sub_props + name=subdomain_name, user_id=user_id, project_id=project_id, props=sub_props, + **self._recon_job_params(), ) stats["subdomains_created"] += 1 @@ -401,9 +435,14 @@ def update_graph_from_ip_recon(self, recon_data: dict, user_id: str, project_id: session.run( """ MERGE (i:IP {address: $addr, user_id: $uid, project_id: $pid}) - SET i += $props + SET i += $props, + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, - addr=ip, uid=user_id, pid=project_id, props=ip_props + addr=ip, uid=user_id, pid=project_id, props=ip_props, + **self._recon_job_params(), ) stats["ips_created"] += 1 diff --git a/graph_db/mixins/recon/http_mixin.py b/graph_db/mixins/recon/http_mixin.py index 1c077dfa..cc0be14e 100644 --- a/graph_db/mixins/recon/http_mixin.py +++ b/graph_db/mixins/recon/http_mixin.py @@ -111,9 +111,14 @@ def update_graph_from_http_probe(self, recon_data: dict, user_id: str, project_i """ MERGE (u:BaseURL {url: $base_url, user_id: $user_id, project_id: $project_id}) SET u += $props, - u.updated_at = datetime() + u.updated_at = datetime(), + u.first_seen = coalesce(u.first_seen, $recon_job_started_at), + u.first_seen_job_id = coalesce(u.first_seen_job_id, $recon_job_id), + u.last_seen = $recon_job_started_at, + u.last_seen_job_id = $recon_job_id """, - base_url=base_url, user_id=user_id, project_id=project_id, props=baseurl_props + base_url=base_url, user_id=user_id, project_id=project_id, props=baseurl_props, + **self._recon_job_params(), ) stats["baseurls_created"] += 1 @@ -160,9 +165,14 @@ def update_graph_from_http_probe(self, recon_data: dict, user_id: str, project_i """ MERGE (e:Endpoint {path: $path, method: 'GET', baseurl: $base_url, user_id: $user_id, project_id: $project_id}) SET e += $props, - e.updated_at = datetime() + e.updated_at = datetime(), + e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id """, - path=path, base_url=base_url, user_id=user_id, project_id=project_id, props=endpoint_props + path=path, base_url=base_url, user_id=user_id, project_id=project_id, props=endpoint_props, + **self._recon_job_params(), ) stats.setdefault("endpoints_created", 0) stats["endpoints_created"] += 1 @@ -207,9 +217,14 @@ def update_graph_from_http_probe(self, recon_data: dict, user_id: str, project_i """ MERGE (c:Certificate {subject_cn: $subject_cn, user_id: $user_id, project_id: $project_id}) SET c += $props, - c.updated_at = datetime() + c.updated_at = datetime(), + c.first_seen = coalesce(c.first_seen, $recon_job_started_at), + c.first_seen_job_id = coalesce(c.first_seen_job_id, $recon_job_id), + c.last_seen = $recon_job_started_at, + c.last_seen_job_id = $recon_job_id """, - subject_cn=subject_cn, user_id=user_id, project_id=project_id, props=cert_props + subject_cn=subject_cn, user_id=user_id, project_id=project_id, props=cert_props, + **self._recon_job_params(), ) stats["certificates_created"] += 1 @@ -257,10 +272,15 @@ def update_graph_from_http_probe(self, recon_data: dict, user_id: str, project_i session.run( """ MERGE (svc:Service {name: $service_name, port_number: $port_number, ip_address: $ip_addr, user_id: $user_id, project_id: $project_id}) - SET svc.updated_at = datetime() + SET svc.updated_at = datetime(), + svc.first_seen = coalesce(svc.first_seen, $recon_job_started_at), + svc.first_seen_job_id = coalesce(svc.first_seen_job_id, $recon_job_id), + svc.last_seen = $recon_job_started_at, + svc.last_seen_job_id = $recon_job_id """, service_name=service_name, port_number=port_number, ip_addr=resolved_ip, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["services_created"] += 1 @@ -282,14 +302,19 @@ def update_graph_from_http_probe(self, recon_data: dict, user_id: str, project_i """ MERGE (p:Port {number: $port_number, protocol: 'tcp', ip_address: $ip_addr, user_id: $user_id, project_id: $project_id}) SET p.state = 'open', - p.updated_at = datetime() + p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id WITH p MATCH (svc:Service {name: $service_name, port_number: $port_number, ip_address: $ip_addr, user_id: $user_id, project_id: $project_id}) MERGE (p)-[:RUNS_SERVICE]->(svc) """, port_number=port_number, ip_addr=resolved_ip, service_name=service_name, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) # Also ensure IP -[:HAS_PORT]-> Port relationship exists @@ -342,21 +367,35 @@ def update_graph_from_http_probe(self, recon_data: dict, user_id: str, project_i """ MERGE (t:Technology {name: $name, version: $version, user_id: $user_id, project_id: $project_id}) SET t += $props, - t.updated_at = datetime() + t.updated_at = datetime(), + t.first_seen = coalesce(t.first_seen, $recon_job_started_at), + t.first_seen_job_id = coalesce(t.first_seen_job_id, $recon_job_id), + t.last_seen = $recon_job_started_at, + t.last_seen_job_id = $recon_job_id """, name=tech_name, version=tech_version, props=tech_props, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) processed_techs.add((tech_name, tech_version)) else: session.run( """ MERGE (t:Technology {name: $name, version: '', user_id: $user_id, project_id: $project_id}) - ON CREATE SET t += $props, t.updated_at = datetime() - ON MATCH SET t.updated_at = datetime() + ON CREATE SET t += $props, t.updated_at = datetime(), + t.first_seen = coalesce(t.first_seen, $recon_job_started_at), + t.first_seen_job_id = coalesce(t.first_seen_job_id, $recon_job_id), + t.last_seen = $recon_job_started_at, + t.last_seen_job_id = $recon_job_id + ON MATCH SET t.updated_at = datetime(), + t.first_seen = coalesce(t.first_seen, $recon_job_started_at), + t.first_seen_job_id = coalesce(t.first_seen_job_id, $recon_job_id), + t.last_seen = $recon_job_started_at, + t.last_seen_job_id = $recon_job_id """, name=tech_name, props=tech_props, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) processed_techs.add((tech_name, None)) stats["technologies_created"] += 1 @@ -415,10 +454,15 @@ def update_graph_from_http_probe(self, recon_data: dict, user_id: str, project_i MERGE (t:Technology {name: $name, user_id: $user_id, project_id: $project_id}) SET t.category = $category, t.source = 'ai-surface-recon', - t.updated_at = datetime() + t.updated_at = datetime(), + t.first_seen = coalesce(t.first_seen, $recon_job_started_at), + t.first_seen_job_id = coalesce(t.first_seen_job_id, $recon_job_id), + t.last_seen = $recon_job_started_at, + t.last_seen_job_id = $recon_job_id """, name=ai_name, category=ai_category, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats.setdefault("ai_technologies_created", 0) stats["ai_technologies_created"] += 1 @@ -480,20 +524,34 @@ def update_graph_from_http_probe(self, recon_data: dict, user_id: str, project_i """ MERGE (t:Technology {name: $name, version: $version, user_id: $user_id, project_id: $project_id}) SET t += $props, - t.updated_at = datetime() + t.updated_at = datetime(), + t.first_seen = coalesce(t.first_seen, $recon_job_started_at), + t.first_seen_job_id = coalesce(t.first_seen_job_id, $recon_job_id), + t.last_seen = $recon_job_started_at, + t.last_seen_job_id = $recon_job_id """, name=tech_name, version=tech_version, props=tech_props, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) else: session.run( """ MERGE (t:Technology {name: $name, version: '', user_id: $user_id, project_id: $project_id}) - ON CREATE SET t += $props, t.updated_at = datetime() - ON MATCH SET t.updated_at = datetime() + ON CREATE SET t += $props, t.updated_at = datetime(), + t.first_seen = coalesce(t.first_seen, $recon_job_started_at), + t.first_seen_job_id = coalesce(t.first_seen_job_id, $recon_job_id), + t.last_seen = $recon_job_started_at, + t.last_seen_job_id = $recon_job_id + ON MATCH SET t.updated_at = datetime(), + t.first_seen = coalesce(t.first_seen, $recon_job_started_at), + t.first_seen_job_id = coalesce(t.first_seen_job_id, $recon_job_id), + t.last_seen = $recon_job_started_at, + t.last_seen_job_id = $recon_job_id """, name=tech_name, props=tech_props, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["technologies_created"] += 1 @@ -545,11 +603,16 @@ def update_graph_from_http_probe(self, recon_data: dict, user_id: str, project_i h.project_id = $project_id, h.is_security_header = $is_security, h.reveals_technology = $reveals_tech, - h.updated_at = datetime() + h.updated_at = datetime(), + h.first_seen = coalesce(h.first_seen, $recon_job_started_at), + h.first_seen_job_id = coalesce(h.first_seen_job_id, $recon_job_id), + h.last_seen = $recon_job_started_at, + h.last_seen_job_id = $recon_job_id """, name=header_name, value=str(header_value), url=url, user_id=user_id, project_id=project_id, - is_security=is_security, reveals_tech=reveals_tech + is_security=is_security, reveals_tech=reveals_tech, + **self._recon_job_params(), ) stats["headers_created"] += 1 @@ -586,12 +649,17 @@ def update_graph_from_http_probe(self, recon_data: dict, user_id: str, project_i SET d.http_probe_timestamp = $scan_timestamp, d.http_probe_live_urls = $live_urls, d.http_probe_technology_count = $tech_count, - d.updated_at = datetime() + d.updated_at = datetime(), + d.first_seen = coalesce(d.first_seen, $recon_job_started_at), + d.first_seen_job_id = coalesce(d.first_seen_job_id, $recon_job_id), + d.last_seen = $recon_job_started_at, + d.last_seen_job_id = $recon_job_id """, root_domain=root_domain, user_id=user_id, project_id=project_id, scan_timestamp=scan_metadata.get("scan_timestamp"), live_urls=summary.get("live_urls", 0), - tech_count=summary.get("technology_count", 0) + tech_count=summary.get("technology_count", 0), + **self._recon_job_params(), ) except Exception as e: stats["errors"].append(f"Domain update failed: {e}") @@ -619,11 +687,16 @@ def update_graph_from_http_probe(self, recon_data: dict, user_id: str, project_i s.status_codes = $status_codes, s.http_live_url_count = $live_count, s.http_probed_at = datetime(), - s.updated_at = datetime() + s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id """, hostname=hostname, user_id=user_id, project_id=project_id, status=http_status, status_codes=status_codes, - live_count=len(live_urls) + live_count=len(live_urls), + **self._recon_job_params(), ) stats["subdomains_updated"] += 1 except Exception as e: @@ -641,9 +714,14 @@ def update_graph_from_http_probe(self, recon_data: dict, user_id: str, project_i WHERE s.status = 'resolved' SET s.status = 'no_http', s.http_probed_at = datetime(), - s.updated_at = datetime() + s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id """, - hosts=list(no_response_hosts), user_id=user_id, project_id=project_id + hosts=list(no_response_hosts), user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) print(f"[+][graph-db] Created {stats['baseurls_created']} BaseURL nodes") diff --git a/graph_db/mixins/recon/js_recon_mixin.py b/graph_db/mixins/recon/js_recon_mixin.py index 35c1336e..b0a069da 100644 --- a/graph_db/mixins/recon/js_recon_mixin.py +++ b/graph_db/mixins/recon/js_recon_mixin.py @@ -39,6 +39,15 @@ def update_graph_from_js_recon(self, recon_data: dict, user_id: str, project_id: scan_ts = js_recon_data.get("scan_metadata", {}).get("scan_timestamp", "") domain_name = recon_data.get('domain', '') + js_finding_seen_query = """ + MERGE (jf:JsReconFinding {id: $id}) + SET jf += $props, + jf.updated_at = datetime(), + jf.first_seen = coalesce(jf.first_seen, $recon_job_started_at), + jf.first_seen_job_id = coalesce(jf.first_seen_job_id, $recon_job_id), + jf.last_seen = $recon_job_started_at, + jf.last_seen_job_id = $recon_job_id + """ def _is_uploaded(source_url: str) -> bool: return source_url.startswith('upload://') @@ -115,8 +124,9 @@ def _filename_from_url(url: str) -> str: } session.run( - "MERGE (jf:JsReconFinding {id: $id}) SET jf += $props, jf.updated_at = datetime()", - id=file_node_id, props=props + js_finding_seen_query, + id=file_node_id, props=props, + **self._recon_job_params(), ) file_node_ids[source_url] = file_node_id stats["file_nodes_created"] += 1 @@ -212,8 +222,9 @@ def _link_to_file(session, node_id: str, node_label: str, rel_type: str, source_ } session.run( - "MERGE (jf:JsReconFinding {id: $id}) SET jf += $props, jf.updated_at = datetime()", - id=node_id, props=props + js_finding_seen_query, + id=node_id, props=props, + **self._recon_job_params(), ) stats["findings_created"] += 1 @@ -250,8 +261,9 @@ def _link_to_file(session, node_id: str, node_label: str, rel_type: str, source_ "discovered_at": scan_ts, } session.run( - "MERGE (jf:JsReconFinding {id: $id}) SET jf += $props, jf.updated_at = datetime()", - id=node_id, props=props + js_finding_seen_query, + id=node_id, props=props, + **self._recon_job_params(), ) stats["findings_created"] += 1 @@ -292,8 +304,9 @@ def _link_to_file(session, node_id: str, node_label: str, rel_type: str, source_ "discovered_at": scan_ts, } session.run( - "MERGE (jf:JsReconFinding {id: $id}) SET jf += $props, jf.updated_at = datetime()", - id=node_id, props=props + js_finding_seen_query, + id=node_id, props=props, + **self._recon_job_params(), ) stats["findings_created"] += 1 @@ -334,8 +347,9 @@ def _link_to_file(session, node_id: str, node_label: str, rel_type: str, source_ "discovered_at": scan_ts, } session.run( - "MERGE (jf:JsReconFinding {id: $id}) SET jf += $props, jf.updated_at = datetime()", - id=node_id, props=props + js_finding_seen_query, + id=node_id, props=props, + **self._recon_job_params(), ) stats["findings_created"] += 1 @@ -377,8 +391,9 @@ def _link_to_file(session, node_id: str, node_label: str, rel_type: str, source_ "discovered_at": scan_ts, } session.run( - "MERGE (jf:JsReconFinding {id: $id}) SET jf += $props, jf.updated_at = datetime()", - id=node_id, props=props + js_finding_seen_query, + id=node_id, props=props, + **self._recon_job_params(), ) stats["findings_created"] += 1 @@ -423,8 +438,9 @@ def _link_to_file(session, node_id: str, node_label: str, rel_type: str, source_ "discovered_at": scan_ts, } session.run( - "MERGE (jf:JsReconFinding {id: $id}) SET jf += $props, jf.updated_at = datetime()", - id=node_id, props=props + js_finding_seen_query, + id=node_id, props=props, + **self._recon_job_params(), ) stats["findings_created"] += 1 @@ -467,8 +483,9 @@ def _link_to_file(session, node_id: str, node_label: str, rel_type: str, source_ "discovered_at": scan_ts, } session.run( - "MERGE (jf:JsReconFinding {id: $id}) SET jf += $props, jf.updated_at = datetime()", - id=node_id, props=props + js_finding_seen_query, + id=node_id, props=props, + **self._recon_job_params(), ) stats["findings_created"] += 1 @@ -520,7 +537,11 @@ def _link_to_file(session, node_id: str, node_label: str, rel_type: str, source_ s.matched_text = $matched_text, s.validation_info = $validation_info, s.discovered_at = $discovered_at, - s.updated_at = datetime() + s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id """, id=node_id, user_id=user_id, project_id=project_id, secret_type=secret.get("name", "unknown"), @@ -535,6 +556,7 @@ def _link_to_file(session, node_id: str, node_label: str, rel_type: str, source_ validation_status=validation.get("status", "unvalidated"), validation_info=json.dumps(validation) if validation else "", discovered_at=scan_ts, + **self._recon_job_params(), ) created_secrets.add(node_id) stats["secrets_created"] += 1 @@ -577,17 +599,26 @@ def _link_to_file(session, node_id: str, node_label: str, rel_type: str, source_ e.category = $category, e.full_url = $full_url, e.endpoint_type = $ep_type, - e.updated_at = datetime() + e.updated_at = datetime(), + e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id ON MATCH SET e.js_recon_source = true, e.endpoint_type = COALESCE(e.endpoint_type, $ep_type), e.full_url = COALESCE(e.full_url, $full_url), - e.updated_at = datetime() + e.updated_at = datetime(), + e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id """, path=path, method=method, baseurl=effective_baseurl, uid=user_id, pid=project_id, category=ep.get("category", "endpoint"), full_url=ep.get("full_url", ""), ep_type=ep.get("type", "rest"), + **self._recon_job_params(), ) stats["endpoints_created"] += 1 @@ -644,8 +675,9 @@ def _link_to_file(session, node_id: str, node_label: str, rel_type: str, source_ "discovered_at": scan_ts, } session.run( - "MERGE (jf:JsReconFinding {id: $id}) SET jf += $props, jf.updated_at = datetime()", - id=node_id, props=props + js_finding_seen_query, + id=node_id, props=props, + **self._recon_job_params(), ) stats["ai_sdk_findings_created"] += 1 @@ -688,12 +720,17 @@ def _link_to_file(session, node_id: str, node_label: str, rel_type: str, source_ OR $needle CONTAINS s.matched_text) SET s.ai_provider = $sdk_name, s.ai_finding_id = $ai_finding_id, - s.updated_at = datetime() + s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id RETURN count(s) AS enriched """, source_url=source_url, uid=user_id, pid=project_id, needle=captured, sdk_name=sdk_name, ai_finding_id=node_id, + **self._recon_job_params(), ).single() if result and result.get("enriched"): stats["ai_sdk_secrets_enriched"] += int(result["enriched"]) diff --git a/graph_db/mixins/recon/port_mixin.py b/graph_db/mixins/recon/port_mixin.py index 4cb0a854..979abd44 100644 --- a/graph_db/mixins/recon/port_mixin.py +++ b/graph_db/mixins/recon/port_mixin.py @@ -69,10 +69,15 @@ def update_graph_from_port_scan(self, recon_data: dict, user_id: str, project_id MERGE (i:IP {address: $address, user_id: $user_id, project_id: $project_id}) SET i.is_cdn = $is_cdn, i.cdn_name = $cdn_name, - i.updated_at = datetime() + i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, address=ip_addr, user_id=user_id, project_id=project_id, - is_cdn=is_cdn, cdn_name=cdn_name + is_cdn=is_cdn, cdn_name=cdn_name, + **self._recon_job_params(), ) stats["ips_updated"] += 1 @@ -94,10 +99,15 @@ def update_graph_from_port_scan(self, recon_data: dict, user_id: str, project_id MERGE (i:IP {address: $address, user_id: $user_id, project_id: $project_id}) SET i.is_cdn = $is_cdn, i.cdn_name = $cdn_name, - i.updated_at = datetime() + i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, address=ip_addr, user_id=user_id, project_id=project_id, - is_cdn=is_cdn, cdn_name=cdn_name + is_cdn=is_cdn, cdn_name=cdn_name, + **self._recon_job_params(), ) except Exception as e: stats["errors"].append(f"IP {ip_addr} update failed: {e}") @@ -118,10 +128,15 @@ def update_graph_from_port_scan(self, recon_data: dict, user_id: str, project_id """ MERGE (p:Port {number: $port_number, protocol: $protocol, ip_address: $ip_addr, user_id: $user_id, project_id: $project_id}) SET p.state = 'open', - p.updated_at = datetime() + p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id """, port_number=port_number, protocol=protocol, ip_addr=ip_addr, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["ports_created"] += 1 @@ -143,10 +158,15 @@ def update_graph_from_port_scan(self, recon_data: dict, user_id: str, project_id session.run( """ MERGE (svc:Service {name: $service_name, port_number: $port_number, ip_address: $ip_addr, user_id: $user_id, project_id: $project_id}) - SET svc.updated_at = datetime() + SET svc.updated_at = datetime(), + svc.first_seen = coalesce(svc.first_seen, $recon_job_started_at), + svc.first_seen_job_id = coalesce(svc.first_seen_job_id, $recon_job_id), + svc.last_seen = $recon_job_started_at, + svc.last_seen_job_id = $recon_job_id """, service_name=service_name, port_number=port_number, ip_addr=ip_addr, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["services_created"] += 1 @@ -178,10 +198,15 @@ def update_graph_from_port_scan(self, recon_data: dict, user_id: str, project_id MERGE (t:Technology {name: $name, user_id: $user_id, project_id: $project_id}) SET t.category = $category, t.source = 'ai-port-catalog', - t.updated_at = datetime() + t.updated_at = datetime(), + t.first_seen = coalesce(t.first_seen, $recon_job_started_at), + t.first_seen_job_id = coalesce(t.first_seen_job_id, $recon_job_id), + t.last_seen = $recon_job_started_at, + t.last_seen_job_id = $recon_job_id """, name=ai_name, category=ai_category, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats.setdefault("ai_technologies_created", 0) stats["ai_technologies_created"] += 1 @@ -231,13 +256,18 @@ def update_graph_from_port_scan(self, recon_data: dict, user_id: str, project_id d.port_scan_type = $scan_type, d.port_scan_ports_config = $ports_config, d.port_scan_total_open_ports = $total_open_ports, - d.updated_at = datetime() + d.updated_at = datetime(), + d.first_seen = coalesce(d.first_seen, $recon_job_started_at), + d.first_seen_job_id = coalesce(d.first_seen_job_id, $recon_job_id), + d.last_seen = $recon_job_started_at, + d.last_seen_job_id = $recon_job_id """, root_domain=root_domain, user_id=user_id, project_id=project_id, scan_timestamp=scan_metadata.get("scan_timestamp"), scan_type=scan_metadata.get("scan_type"), ports_config=scan_metadata.get("ports_config"), - total_open_ports=port_scan_data.get("summary", {}).get("total_open_ports", 0) + total_open_ports=port_scan_data.get("summary", {}).get("total_open_ports", 0), + **self._recon_job_params(), ) except Exception as e: stats["errors"].append(f"Domain update failed: {e}") @@ -305,11 +335,16 @@ def update_graph_from_nmap(self, recon_data: dict, user_id: str, project_id: str p.version = $version, p.cpe = $cpe, p.nmap_scanned = true, - p.updated_at = datetime() + p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id """, port_number=port_number, ip_addr=ip_addr, product=product, version=version, cpe=cpe, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["ports_enriched"] += 1 @@ -321,11 +356,16 @@ def update_graph_from_nmap(self, recon_data: dict, user_id: str, project_id: str SET svc.product = $product, svc.version = $version, svc.cpe = $cpe, - svc.updated_at = datetime() + svc.updated_at = datetime(), + svc.first_seen = coalesce(svc.first_seen, $recon_job_started_at), + svc.first_seen_job_id = coalesce(svc.first_seen_job_id, $recon_job_id), + svc.last_seen = $recon_job_started_at, + svc.last_seen_job_id = $recon_job_id """, port_number=port_number, ip_addr=ip_addr, product=product, version=version, cpe=cpe, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["services_enriched"] += 1 @@ -336,11 +376,16 @@ def update_graph_from_nmap(self, recon_data: dict, user_id: str, project_id: str """ MATCH (svc:Service {port_number: $port_number, ip_address: $ip_addr, user_id: $user_id, project_id: $project_id}) SET svc.ai_runtime_version = $ai_runtime_version, - svc.updated_at = datetime() + svc.updated_at = datetime(), + svc.first_seen = coalesce(svc.first_seen, $recon_job_started_at), + svc.first_seen_job_id = coalesce(svc.first_seen_job_id, $recon_job_id), + svc.last_seen = $recon_job_started_at, + svc.last_seen_job_id = $recon_job_id """, port_number=port_number, ip_addr=ip_addr, ai_runtime_version=ai_runtime_version, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats.setdefault("ai_runtime_versions_set", 0) stats["ai_runtime_versions_set"] += 1 @@ -373,11 +418,16 @@ def update_graph_from_nmap(self, recon_data: dict, user_id: str, project_id: str SET t.version = $version, t.source = 'nmap', t.cpe = $cpe, - t.updated_at = datetime() + t.updated_at = datetime(), + t.first_seen = coalesce(t.first_seen, $recon_job_started_at), + t.first_seen_job_id = coalesce(t.first_seen_job_id, $recon_job_id), + t.last_seen = $recon_job_started_at, + t.last_seen_job_id = $recon_job_id """, name=tech_name, version=version or "", cpe=svc.get("cpe", ""), - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["technologies_created"] += 1 @@ -437,12 +487,17 @@ def update_graph_from_nmap(self, recon_data: dict, user_id: str, project_id: str v.output = $output, v.state = $state, v.cve_id = $cve_id, - v.updated_at = datetime() + v.updated_at = datetime(), + v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id), + v.last_seen = $recon_job_started_at, + v.last_seen_job_id = $recon_job_id """, name=script_id, ip_addr=ip_addr, port_number=port_number, severity=severity, output=output[:2000], state=state, cve_id=cve_id, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["nse_vulns_created"] += 1 diff --git a/graph_db/mixins/recon/resource_mixin.py b/graph_db/mixins/recon/resource_mixin.py index c9a18706..83c3d9ed 100644 --- a/graph_db/mixins/recon/resource_mixin.py +++ b/graph_db/mixins/recon/resource_mixin.py @@ -134,7 +134,11 @@ def is_in_scope(base_url: str) -> bool: END, e.ai_interface_type = COALESCE($ai_interface_type, e.ai_interface_type), e.is_ai_rag_ingest = COALESCE($is_ai_rag_ingest, e.is_ai_rag_ingest), - e.updated_at = datetime() + e.updated_at = datetime(), + e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id """, path=path, method=method, baseurl=base_url, user_id=user_id, project_id=project_id, @@ -147,6 +151,7 @@ def is_in_scope(base_url: str) -> bool: sources=ep_sources, ai_interface_type=ai_interface_type, is_ai_rag_ingest=is_ai_rag_ingest, + **self._recon_job_params(), ) stats["endpoints_created"] += 1 created_endpoints.add(endpoint_key) @@ -156,13 +161,22 @@ def is_in_scope(base_url: str) -> bool: """ MERGE (bu:BaseURL {url: $baseurl, user_id: $user_id, project_id: $project_id}) ON CREATE SET bu.source = 'resource_enum', - bu.updated_at = datetime() + bu.updated_at = datetime(), + bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at), + bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id), + bu.last_seen = $recon_job_started_at, + bu.last_seen_job_id = $recon_job_id + ON MATCH SET bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at), + bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id), + bu.last_seen = $recon_job_started_at, + bu.last_seen_job_id = $recon_job_id WITH bu MATCH (e:Endpoint {path: $path, method: $method, baseurl: $baseurl, user_id: $user_id, project_id: $project_id}) MERGE (bu)-[:HAS_ENDPOINT]->(e) """, baseurl=base_url, path=path, method=method, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["relationships_created"] += 1 @@ -198,7 +212,11 @@ def is_in_scope(base_url: str) -> bool: p.source = 'resource_enum', p.is_ai_prompt_injectable = COALESCE($is_ai_prompt_injectable, p.is_ai_prompt_injectable), p.ai_tool_arg_path = COALESCE($ai_tool_arg_path, p.ai_tool_arg_path), - p.updated_at = datetime() + p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id """, name=param_name, position="query", endpoint_path=path, baseurl=base_url, user_id=user_id, project_id=project_id, @@ -207,6 +225,7 @@ def is_in_scope(base_url: str) -> bool: sample_values=sample_values[:5], # Limit sample values is_ai_prompt_injectable=is_ai_prompt_injectable, ai_tool_arg_path=ai_tool_arg_path, + **self._recon_job_params(), ) stats["parameters_created"] += 1 created_parameters.add(param_key) @@ -250,7 +269,11 @@ def is_in_scope(base_url: str) -> bool: p.source = 'resource_enum', p.is_ai_prompt_injectable = COALESCE($is_ai_prompt_injectable, p.is_ai_prompt_injectable), p.ai_tool_arg_path = COALESCE($ai_tool_arg_path, p.ai_tool_arg_path), - p.updated_at = datetime() + p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id """, name=param_name, position="body", endpoint_path=path, baseurl=base_url, user_id=user_id, project_id=project_id, @@ -260,6 +283,7 @@ def is_in_scope(base_url: str) -> bool: required=param.get("required", False), is_ai_prompt_injectable=is_ai_prompt_injectable, ai_tool_arg_path=ai_tool_arg_path, + **self._recon_job_params(), ) stats["parameters_created"] += 1 created_parameters.add(param_key) @@ -339,14 +363,19 @@ def is_in_scope(base_url: str) -> bool: e.form_enctype = $enctype, e.form_found_at_pages = $found_at_pages, e.form_input_names = $input_names, - e.form_count = $form_count + e.form_count = $form_count, + e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id """, path=path, method=method, baseurl=baseurl, user_id=user_id, project_id=project_id, enctype=form_info["enctype"], found_at_pages=list(form_info["found_at_pages"]), input_names=list(form_info["input_names"]), - form_count=len(form_info["found_at_pages"]) + form_count=len(form_info["found_at_pages"]), + **self._recon_job_params(), ) stats["forms_created"] += 1 @@ -400,7 +429,11 @@ def is_in_scope(base_url: str) -> bool: s.base_url = $base_url, s.sample = $sample, s.discovered_at = $discovered_at, - s.updated_at = datetime() + s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id WITH s MATCH (bu:BaseURL {url: $base_url, user_id: $user_id, project_id: $project_id}) MERGE (bu)-[:HAS_SECRET]->(s) @@ -408,7 +441,8 @@ def is_in_scope(base_url: str) -> bool: id=node_id, user_id=user_id, project_id=project_id, secret_type=secret_type, severity=severity, source_url=source_url, base_url=base_url, - sample=sample, discovered_at=scan_ts + sample=sample, discovered_at=scan_ts, + **self._recon_job_params(), ) created_secrets.add(node_id) stats["secrets_created"] += 1 @@ -433,13 +467,18 @@ def is_in_scope(base_url: str) -> bool: d.resource_enum_total_endpoints = $total_endpoints, d.resource_enum_total_parameters = $total_parameters, d.resource_enum_total_forms = $total_forms, - d.updated_at = datetime() + d.updated_at = datetime(), + d.first_seen = coalesce(d.first_seen, $recon_job_started_at), + d.first_seen_job_id = coalesce(d.first_seen_job_id, $recon_job_id), + d.last_seen = $recon_job_started_at, + d.last_seen_job_id = $recon_job_id """, root_domain=root_domain, user_id=user_id, project_id=project_id, scan_timestamp=resource_enum_data.get("scan_metadata", {}).get("scan_timestamp"), total_endpoints=summary.get("total_endpoints", 0), total_parameters=summary.get("total_parameters", 0), - total_forms=summary.get("total_forms", 0) + total_forms=summary.get("total_forms", 0), + **self._recon_job_params(), ) except Exception as e: stats["errors"].append(f"Domain update failed: {e}") diff --git a/graph_db/mixins/recon/takeover_mixin.py b/graph_db/mixins/recon/takeover_mixin.py index 2842ebb0..c712e3f9 100644 --- a/graph_db/mixins/recon/takeover_mixin.py +++ b/graph_db/mixins/recon/takeover_mixin.py @@ -25,7 +25,7 @@ verdict confirmed | likely | manual_review evidence raw response / fingerprint hit excerpt tool_raw JSON-encoded raw output per tool - first_seen, last_seen ISO timestamps + detected_at tool-native detection timestamp """ from __future__ import annotations @@ -110,21 +110,27 @@ def update_graph_from_subdomain_takeover( "host": hostname, "is_dast_finding": False, "tool_raw": tool_raw, - "last_seen": detected_at, + "detected_at": detected_at, } # Remove None values so MERGE's SET += doesn't wipe existing props vuln_props = {k: v for k, v in vuln_props.items() if v is not None} - # Merge the Vulnerability node (set last_seen on every run, - # first_seen only on create). + # Merge the Vulnerability node. Job-level first/last seen + # metadata is stamped from the active recon job context. session.run( """ MERGE (v:Vulnerability {id: $id}) - ON CREATE SET v.first_seen = $detected_at + ON CREATE SET v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id) SET v += $props, - v.updated_at = datetime() + v.updated_at = datetime(), + v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id), + v.last_seen = $recon_job_started_at, + v.last_seen_job_id = $recon_job_id """, - id=vuln_id, props=vuln_props, detected_at=detected_at, + id=vuln_id, props=vuln_props, + **self._recon_job_params(), ) stats["vulnerabilities_created"] += 1 @@ -166,13 +172,22 @@ def update_graph_from_subdomain_takeover( """ MERGE (s:Subdomain {name: $hostname, user_id: $uid, project_id: $pid}) ON CREATE SET s.source = 'takeover_scan', - s.created_at = datetime() - SET s.updated_at = datetime() + s.created_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id + SET s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id WITH s MATCH (v:Vulnerability {id: $id}) MERGE (s)-[:HAS_VULNERABILITY]->(v) """, hostname=hostname, uid=user_id, pid=project_id, id=vuln_id, + **self._recon_job_params(), ) stats["relationships_created"] += 1 diff --git a/graph_db/mixins/recon/user_input_mixin.py b/graph_db/mixins/recon/user_input_mixin.py index cbd4d507..efdb2098 100644 --- a/graph_db/mixins/recon/user_input_mixin.py +++ b/graph_db/mixins/recon/user_input_mixin.py @@ -37,7 +37,11 @@ def create_user_input_node(self, domain: str, user_input_data: dict, user_id: st ui.status = 'running', ui.created_at = datetime(), ui.user_id = $user_id, - ui.project_id = $project_id + ui.project_id = $project_id, + ui.first_seen = coalesce(ui.first_seen, $recon_job_started_at), + ui.first_seen_job_id = coalesce(ui.first_seen_job_id, $recon_job_id), + ui.last_seen = $recon_job_started_at, + ui.last_seen_job_id = $recon_job_id """, id=node_id, input_type=user_input_data.get("input_type", "subdomains"), @@ -45,18 +49,28 @@ def create_user_input_node(self, domain: str, user_input_data: dict, user_id: st tool_id=user_input_data.get("tool_id", ""), user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) # Connect to Domain node (create Domain if needed via MERGE) session.run( """ MERGE (d:Domain {name: $domain, user_id: $user_id, project_id: $project_id}) - ON CREATE SET d.updated_at = datetime() + ON CREATE SET d.updated_at = datetime(), + d.first_seen = coalesce(d.first_seen, $recon_job_started_at), + d.first_seen_job_id = coalesce(d.first_seen_job_id, $recon_job_id), + d.last_seen = $recon_job_started_at, + d.last_seen_job_id = $recon_job_id + ON MATCH SET d.first_seen = coalesce(d.first_seen, $recon_job_started_at), + d.first_seen_job_id = coalesce(d.first_seen_job_id, $recon_job_id), + d.last_seen = $recon_job_started_at, + d.last_seen_job_id = $recon_job_id WITH d MATCH (ui:UserInput {id: $ui_id}) MERGE (d)-[:HAS_USER_INPUT]->(ui) """, domain=domain, user_id=user_id, project_id=project_id, ui_id=node_id, + **self._recon_job_params(), ) print(f"[+][graph-db] Created UserInput node {node_id} for {domain}") @@ -128,10 +142,19 @@ def update_graph_from_partial_discovery( session.run( """ MERGE (d:Domain {name: $name, user_id: $user_id, project_id: $project_id}) - ON CREATE SET d.updated_at = datetime() - ON MATCH SET d.updated_at = datetime() + ON CREATE SET d.updated_at = datetime(), + d.first_seen = coalesce(d.first_seen, $recon_job_started_at), + d.first_seen_job_id = coalesce(d.first_seen_job_id, $recon_job_id), + d.last_seen = $recon_job_started_at, + d.last_seen_job_id = $recon_job_id + ON MATCH SET d.updated_at = datetime(), + d.first_seen = coalesce(d.first_seen, $recon_job_started_at), + d.first_seen_job_id = coalesce(d.first_seen_job_id, $recon_job_id), + d.last_seen = $recon_job_started_at, + d.last_seen_job_id = $recon_job_id """, name=domain, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) subdomain_dns = dns_data.get("subdomains", {}) if dns_data else {} @@ -165,10 +188,15 @@ def update_graph_from_partial_discovery( SET s.has_dns_records = $has_records, s.status = coalesce(s.status, $status), s.discovered_at = coalesce(s.discovered_at, datetime()), - s.updated_at = datetime() + s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id """, name=subdomain, user_id=user_id, project_id=project_id, has_records=has_records, status=sub_status, + **self._recon_job_params(), ) stats["subdomains_total"] += 1 if existed: @@ -221,10 +249,15 @@ def update_graph_from_partial_discovery( """ MERGE (i:IP {address: $address, user_id: $uid, project_id: $pid}) SET i.version = $version, - i.updated_at = datetime() + i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, address=ip_addr, uid=user_id, pid=project_id, version=ip_version, + **self._recon_job_params(), ) stats["ips_total"] += 1 if not ip_exists: @@ -270,10 +303,15 @@ def update_graph_from_partial_discovery( session.run( """ MERGE (dns:DNSRecord {type: $type, value: $value, subdomain: $sub, user_id: $uid, project_id: $pid}) - SET dns.updated_at = datetime() + SET dns.updated_at = datetime(), + dns.first_seen = coalesce(dns.first_seen, $recon_job_started_at), + dns.first_seen_job_id = coalesce(dns.first_seen_job_id, $recon_job_id), + dns.last_seen = $recon_job_started_at, + dns.last_seen_job_id = $recon_job_id """, type=record_type, value=str(value), sub=subdomain, uid=user_id, pid=project_id, + **self._recon_job_params(), ) stats["dns_records_created"] += 1 diff --git a/graph_db/mixins/recon/vhost_sni_mixin.py b/graph_db/mixins/recon/vhost_sni_mixin.py index 227ba6aa..82d3f07f 100644 --- a/graph_db/mixins/recon/vhost_sni_mixin.py +++ b/graph_db/mixins/recon/vhost_sni_mixin.py @@ -28,7 +28,7 @@ observed_size body size returned with vhost lie applied size_delta observed_size - baseline_size internal_pattern_match matched internal-keyword (e.g. "admin"), or None - first_seen, last_seen ISO timestamps + detected_at tool-native detection timestamp Properties enriched on existing Subdomain nodes: vhost_tested, vhost_hidden, vhost_routing_layer, vhost_status_code, @@ -100,10 +100,15 @@ def update_graph_from_vhost_sni( res = session.run( """ MATCH (i:IP {address: $addr, user_id: $uid, project_id: $pid}) - SET i += $props + SET i += $props, + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id RETURN count(i) AS matched """, addr=ip_addr, uid=user_id, pid=project_id, props=ip_props, + **self._recon_job_params(), ) if res.single()["matched"] > 0: stats["ips_enriched"] += 1 @@ -149,18 +154,24 @@ def update_graph_from_vhost_sni( "internal_pattern_match": finding.get("internal_pattern_match"), "matched_at": _build_url(hostname, port, finding.get("scheme") or "https"), "is_dast_finding": False, - "last_seen": detected_at, + "detected_at": detected_at, } vuln_props = {k: v for k, v in vuln_props.items() if v is not None} session.run( """ MERGE (v:Vulnerability {id: $id}) - ON CREATE SET v.first_seen = $detected_at + ON CREATE SET v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id) SET v += $props, - v.updated_at = datetime() + v.updated_at = datetime(), + v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id), + v.last_seen = $recon_job_started_at, + v.last_seen_job_id = $recon_job_id """, - id=vuln_id, props=vuln_props, detected_at=detected_at, + id=vuln_id, props=vuln_props, + **self._recon_job_params(), ) stats["vulnerabilities_created"] += 1 @@ -180,15 +191,24 @@ def update_graph_from_vhost_sni( """ MERGE (s:Subdomain {name: $hostname, user_id: $uid, project_id: $pid}) ON CREATE SET s.source = 'vhost_sni_enum', - s.created_at = datetime() + s.created_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id SET s += $sprops, - s.updated_at = datetime() + s.updated_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id WITH s MATCH (v:Vulnerability {id: $id}) MERGE (s)-[:HAS_VULNERABILITY]->(v) """, hostname=hostname, uid=user_id, pid=project_id, id=vuln_id, sprops=sub_props, + **self._recon_job_params(), ) stats["subdomains_enriched"] += 1 stats["relationships_created"] += 1 @@ -271,17 +291,34 @@ def update_graph_from_vhost_sni( b.created_at = datetime(), b.scheme = $scheme, b.host = $host, - b.port = $port - SET b.updated_at = datetime() + b.port = $port, + b.first_seen = coalesce(b.first_seen, $recon_job_started_at), + b.first_seen_job_id = coalesce(b.first_seen_job_id, $recon_job_id), + b.last_seen = $recon_job_started_at, + b.last_seen_job_id = $recon_job_id + SET b.updated_at = datetime(), + b.first_seen = coalesce(b.first_seen, $recon_job_started_at), + b.first_seen_job_id = coalesce(b.first_seen_job_id, $recon_job_id), + b.last_seen = $recon_job_started_at, + b.last_seen_job_id = $recon_job_id WITH b MERGE (s:Subdomain {name: $host, user_id: $uid, project_id: $pid}) ON CREATE SET s.source = 'vhost_sni_enum', - s.created_at = datetime() + s.created_at = datetime(), + s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id + ON MATCH SET s.first_seen = coalesce(s.first_seen, $recon_job_started_at), + s.first_seen_job_id = coalesce(s.first_seen_job_id, $recon_job_id), + s.last_seen = $recon_job_started_at, + s.last_seen_job_id = $recon_job_id MERGE (s)-[:HAS_BASEURL]->(b) RETURN count(b) AS created """, url=url, uid=user_id, pid=project_id, scheme=_scheme(url), host=hostname, port=_port(url), + **self._recon_job_params(), ) if res.single()["created"] > 0: stats["baseurls_created"] += 1 diff --git a/graph_db/mixins/recon/vuln_mixin.py b/graph_db/mixins/recon/vuln_mixin.py index ddcd7911..345710e6 100644 --- a/graph_db/mixins/recon/vuln_mixin.py +++ b/graph_db/mixins/recon/vuln_mixin.py @@ -269,12 +269,17 @@ def is_in_scope(hostname: str) -> bool: e.has_parameters = $has_parameters, e.full_url = $full_url, e.source = 'katana_crawl', - e.updated_at = datetime() + e.updated_at = datetime(), + e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id """, path=path, method=method, baseurl=base_url, user_id=user_id, project_id=project_id, has_parameters=has_parameters, - full_url=dast_url.split('?')[0] # URL without query params + full_url=dast_url.split('?')[0], # URL without query params + **self._recon_job_params(), ) stats["endpoints_created"] += 1 created_endpoints.add(endpoint_key) @@ -285,13 +290,22 @@ def is_in_scope(hostname: str) -> bool: """ MERGE (bu:BaseURL {url: $baseurl, user_id: $user_id, project_id: $project_id}) ON CREATE SET bu.source = 'resource_enum', - bu.updated_at = datetime() + bu.updated_at = datetime(), + bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at), + bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id), + bu.last_seen = $recon_job_started_at, + bu.last_seen_job_id = $recon_job_id + ON MATCH SET bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at), + bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id), + bu.last_seen = $recon_job_started_at, + bu.last_seen_job_id = $recon_job_id WITH bu MATCH (e:Endpoint {path: $path, method: $method, baseurl: $baseurl, user_id: $user_id, project_id: $project_id}) MERGE (bu)-[:HAS_ENDPOINT]->(e) """, baseurl=base_url, path=path, method=method, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["relationships_created"] += 1 @@ -311,11 +325,16 @@ def is_in_scope(hostname: str) -> bool: p.project_id = $project_id, p.sample_value = $sample_value, p.is_injectable = false, - p.updated_at = datetime() + p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id """, name=param_name, position="query", endpoint_path=path, baseurl=base_url, user_id=user_id, project_id=project_id, - sample_value=sample_value + sample_value=sample_value, + **self._recon_job_params(), ) stats["parameters_created"] += 1 created_parameters.add(param_key) @@ -439,9 +458,14 @@ def is_in_scope(hostname: str) -> bool: """ MERGE (v:Vulnerability {id: $id}) SET v += $props, - v.updated_at = datetime() + v.updated_at = datetime(), + v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id), + v.last_seen = $recon_job_started_at, + v.last_seen_job_id = $recon_job_id """, - id=vuln_id, props=vuln_props + id=vuln_id, props=vuln_props, + **self._recon_job_params(), ) stats["vulnerabilities_created"] += 1 @@ -463,10 +487,15 @@ def is_in_scope(hostname: str) -> bool: e.project_id = $project_id, e.has_parameters = true, e.source = 'vuln_scan', - e.updated_at = datetime() + e.updated_at = datetime(), + e.first_seen = coalesce(e.first_seen, $recon_job_started_at), + e.first_seen_job_id = coalesce(e.first_seen_job_id, $recon_job_id), + e.last_seen = $recon_job_started_at, + e.last_seen_job_id = $recon_job_id """, path=vuln_path, method=fuzzing_method, baseurl=vuln_base_url, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["endpoints_created"] += 1 created_endpoints.add(endpoint_key) @@ -476,13 +505,22 @@ def is_in_scope(hostname: str) -> bool: """ MERGE (bu:BaseURL {url: $baseurl, user_id: $user_id, project_id: $project_id}) ON CREATE SET bu.source = 'vuln_scan', - bu.updated_at = datetime() + bu.updated_at = datetime(), + bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at), + bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id), + bu.last_seen = $recon_job_started_at, + bu.last_seen_job_id = $recon_job_id + ON MATCH SET bu.first_seen = coalesce(bu.first_seen, $recon_job_started_at), + bu.first_seen_job_id = coalesce(bu.first_seen_job_id, $recon_job_id), + bu.last_seen = $recon_job_started_at, + bu.last_seen_job_id = $recon_job_id WITH bu MATCH (e:Endpoint {path: $path, method: $method, baseurl: $baseurl, user_id: $user_id, project_id: $project_id}) MERGE (bu)-[:HAS_ENDPOINT]->(e) """, baseurl=vuln_base_url, path=vuln_path, method=fuzzing_method, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) stats["relationships_created"] += 1 @@ -512,10 +550,15 @@ def is_in_scope(hostname: str) -> bool: SET p.user_id = $user_id, p.project_id = $project_id, p.is_injectable = true, - p.updated_at = datetime() + p.updated_at = datetime(), + p.first_seen = coalesce(p.first_seen, $recon_job_started_at), + p.first_seen_job_id = coalesce(p.first_seen_job_id, $recon_job_id), + p.last_seen = $recon_job_started_at, + p.last_seen_job_id = $recon_job_id """, name=fuzzing_param, position=fuzzing_position, endpoint_path=vuln_path, baseurl=vuln_base_url, - user_id=user_id, project_id=project_id + user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) if param_key not in created_parameters: @@ -780,9 +823,14 @@ def is_in_scope(hostname: str) -> bool: """ MERGE (v:Vulnerability {id: $id}) SET v += $props, - v.updated_at = datetime() + v.updated_at = datetime(), + v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id), + v.last_seen = $recon_job_started_at, + v.last_seen_job_id = $recon_job_id """, - id=vuln_id, props=vuln_props + id=vuln_id, props=vuln_props, + **self._recon_job_params(), ) security_checks_created += 1 stats["vulnerabilities_created"] += 1 @@ -793,9 +841,14 @@ def is_in_scope(hostname: str) -> bool: session.run( """ MERGE (i:IP {address: $address, user_id: $user_id, project_id: $project_id}) - SET i.updated_at = datetime() + SET i.updated_at = datetime(), + i.first_seen = coalesce(i.first_seen, $recon_job_started_at), + i.first_seen_job_id = coalesce(i.first_seen_job_id, $recon_job_id), + i.last_seen = $recon_job_started_at, + i.last_seen_job_id = $recon_job_id """, - address=ip_address, user_id=user_id, project_id=project_id + address=ip_address, user_id=user_id, project_id=project_id, + **self._recon_job_params(), ) session.run( @@ -900,9 +953,14 @@ def is_in_scope(hostname: str) -> bool: """ MERGE (v:Vulnerability {id: $id}) SET v += $props, - v.updated_at = datetime() + v.updated_at = datetime(), + v.first_seen = coalesce(v.first_seen, $recon_job_started_at), + v.first_seen_job_id = coalesce(v.first_seen_job_id, $recon_job_id), + v.last_seen = $recon_job_started_at, + v.last_seen_job_id = $recon_job_id """, - id=vuln_id, props=vuln_props + id=vuln_id, props=vuln_props, + **self._recon_job_params(), ) security_checks_created += 1 stats["vulnerabilities_created"] += 1 @@ -1058,7 +1116,11 @@ def is_in_scope(hostname: str) -> bool: d.vuln_scan_high_count = $high_count, d.vuln_scan_medium_count = $medium_count, d.vuln_scan_low_count = $low_count, - d.updated_at = datetime() + d.updated_at = datetime(), + d.first_seen = coalesce(d.first_seen, $recon_job_started_at), + d.first_seen_job_id = coalesce(d.first_seen_job_id, $recon_job_id), + d.last_seen = $recon_job_started_at, + d.last_seen_job_id = $recon_job_id """, root_domain=root_domain, user_id=user_id, project_id=project_id, scan_timestamp=scan_metadata.get("scan_timestamp"), @@ -1068,7 +1130,8 @@ def is_in_scope(hostname: str) -> bool: critical_count=summary.get("critical", 0), high_count=summary.get("high", 0), medium_count=summary.get("medium", 0), - low_count=summary.get("low", 0) + low_count=summary.get("low", 0), + **self._recon_job_params(), ) except Exception as e: stats["errors"].append(f"Domain update failed: {e}") diff --git a/recon/main.py b/recon/main.py index e62f331b..1c621978 100644 --- a/recon/main.py +++ b/recon/main.py @@ -20,6 +20,7 @@ import sys import json import copy +import os from pathlib import Path from datetime import datetime import threading @@ -44,6 +45,7 @@ USE_BRUTEFORCE_FOR_SUBDOMAINS = _settings['USE_BRUTEFORCE_FOR_SUBDOMAINS'] SCAN_MODULES = _settings['SCAN_MODULES'] UPDATE_GRAPH_DB = _settings['UPDATE_GRAPH_DB'] +DELETE_GRAPH_BEFORE_RECON = os.environ.get("RECON_DELETE_GRAPH", "false").strip().lower() in {"1", "true", "yes", "on"} USER_ID = _settings['USER_ID'] PROJECT_ID = _settings['PROJECT_ID'] VERIFY_DOMAIN_OWNERSHIP = _settings['VERIFY_DOMAIN_OWNERSHIP'] @@ -1564,7 +1566,7 @@ def main(): print("═" * 63) # Clear previous graph data - if UPDATE_GRAPH_DB: + if UPDATE_GRAPH_DB and DELETE_GRAPH_BEFORE_RECON: print("[*][graph-db] Clearing previous graph data for this project...") try: from graph_db import Neo4jClient @@ -1576,6 +1578,8 @@ def main(): print("[!][graph-db] Could not connect to Neo4j - skipping clear\n") except Exception as e: print(f"[!][graph-db] Failed to clear previous graph data: {e}\n") + elif UPDATE_GRAPH_DB: + print("[*][graph-db] Preserving existing graph data for this project") run_ip_recon(TARGET_IPS, _settings) @@ -1653,7 +1657,7 @@ def main(): return 1 # Clear previous graph data for this project before starting new scan - if UPDATE_GRAPH_DB: + if UPDATE_GRAPH_DB and DELETE_GRAPH_BEFORE_RECON: print("[*][graph-db] Clearing previous graph data for this project...") try: from graph_db import Neo4jClient @@ -1665,6 +1669,8 @@ def main(): print("[!][graph-db] Could not connect to Neo4j - skipping clear\n") except Exception as e: print(f"[!][graph-db] Failed to clear previous graph data: {e}\n") + elif UPDATE_GRAPH_DB: + print("[*][graph-db] Preserving existing graph data for this project") # Check anonymity status if Tor is enabled if USE_TOR_FOR_RECON: diff --git a/recon/tests/test_graph_delete_gate.py b/recon/tests/test_graph_delete_gate.py new file mode 100644 index 00000000..063f46fa --- /dev/null +++ b/recon/tests/test_graph_delete_gate.py @@ -0,0 +1,16 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MAIN = ROOT / "main.py" + + +def test_main_reads_recon_delete_graph_env(): + src = MAIN.read_text(encoding="utf-8") + assert "RECON_DELETE_GRAPH" in src + assert "DELETE_GRAPH_BEFORE_RECON" in src + + +def test_clear_project_data_is_guarded_by_delete_flag(): + src = MAIN.read_text(encoding="utf-8") + assert "if UPDATE_GRAPH_DB and DELETE_GRAPH_BEFORE_RECON:" in src + assert "Preserving existing graph data" in src diff --git a/recon_orchestrator/api.py b/recon_orchestrator/api.py index 97bf7b61..2ef3651f 100644 --- a/recon_orchestrator/api.py +++ b/recon_orchestrator/api.py @@ -340,6 +340,7 @@ async def start_recon(project_id: str, request: ReconStartRequest): webapp_api_url=request.webapp_api_url, recon_path=RECON_PATH, custom_templates_path=CUSTOM_TEMPLATES_PATH, + delete_graph=request.delete_graph, ) return state except ValueError as e: diff --git a/recon_orchestrator/container_manager.py b/recon_orchestrator/container_manager.py index fd8997eb..dedc90df 100644 --- a/recon_orchestrator/container_manager.py +++ b/recon_orchestrator/container_manager.py @@ -7,7 +7,7 @@ import re import uuid from datetime import datetime, timedelta, timezone -from pathlib import Path +from pathlib import Path, PureWindowsPath from typing import AsyncGenerator, Optional import docker @@ -101,6 +101,27 @@ def _get_container_name(self, project_id: str) -> str: safe_id = re.sub(r'[^a-zA-Z0-9_.-]', '_', project_id) return f"redamon-recon-{safe_id}" + @staticmethod + def _sibling_host_path(path: str, sibling: str) -> str: + """Return a sibling host path without treating Windows drive paths as relative.""" + if re.match(r"^[A-Za-z]:[\\/]", path) or path.startswith("\\\\"): + return str(PureWindowsPath(path).parent / sibling) + return str(Path(path).parent / sibling) + + def _format_recon_job_id(self, prefix: str, started_at: datetime, unique_value: str) -> str: + """Format graph job metadata ID for recon containers.""" + started_utc = started_at.astimezone(timezone.utc) + stamp = started_utc.strftime("%Y%m%dT%H%M%SZ") + suffix = "".join(ch for ch in unique_value if ch.isalnum())[:8] + if not suffix: + suffix = uuid.uuid4().hex[:8] + return f"{prefix}-{stamp}-{suffix}" + + def _format_recon_job_started_at(self, started_at: datetime) -> str: + """Format graph job start time as UTC ISO-8601 without microseconds.""" + started_utc = started_at.astimezone(timezone.utc).replace(microsecond=0) + return started_utc.isoformat().replace("+00:00", "Z") + async def get_status(self, project_id: str) -> ReconState: """Get current status of a recon process""" if project_id in self.running_states: @@ -168,6 +189,7 @@ async def start_recon( webapp_api_url: str, recon_path: str, custom_templates_path: str = "", + delete_graph: bool = False, ) -> ReconState: """Start a recon container for a project""" @@ -196,6 +218,9 @@ async def start_recon( started_at=datetime.now(timezone.utc), ) self.running_states[project_id] = state + job_unique = uuid.uuid4().hex + recon_job_id = self._format_recon_job_id("full", state.started_at, job_unique) + recon_job_started_at = self._format_recon_job_started_at(state.started_at) try: # Ensure recon image exists @@ -221,6 +246,9 @@ async def start_recon( "USER_ID": user_id, "WEBAPP_API_URL": webapp_api_url, "UPDATE_GRAPH_DB": "true", + "RECON_DELETE_GRAPH": "true" if delete_graph else "false", + "RECON_JOB_ID": recon_job_id, + "RECON_JOB_STARTED_AT": recon_job_started_at, # HOST_RECON_OUTPUT_PATH: Required for nested Docker containers (naabu, httpx, etc.) # These run as sibling containers and need host paths for volume mounts "HOST_RECON_OUTPUT_PATH": f"{recon_path}/output", @@ -240,7 +268,7 @@ async def start_recon( # Note: rw needed because output/data are subdirectories f"{recon_path}": {"bind": "/app/recon", "mode": "rw"}, # Mount graph_db module - f"{Path(recon_path).parent}/graph_db": {"bind": "/app/graph_db", "mode": "ro"}, + self._sibling_host_path(recon_path, "graph_db"): {"bind": "/app/graph_db", "mode": "ro"}, # Mount /tmp for Docker-in-Docker temp files (avoids spaces in paths) "/tmp/redamon": {"bind": "/tmp/redamon", "mode": "rw"}, # JS Recon shared volumes with webapp @@ -707,6 +735,8 @@ async def start_partial_recon( started_at=datetime.now(timezone.utc), ) self.partial_recon_states.setdefault(project_id, {})[run_id] = state + recon_job_id = self._format_recon_job_id("partial", state.started_at, run_id) + recon_job_started_at = self._format_recon_job_started_at(state.started_at) try: # Ensure recon image exists @@ -738,6 +768,8 @@ async def start_partial_recon( "PARTIAL_RECON_CONFIG": f"/tmp/redamon/partial_{project_id}_{run_id}.json", "PARTIAL_RECON_RUN_ID": run_id, "UPDATE_GRAPH_DB": "true", + "RECON_JOB_ID": recon_job_id, + "RECON_JOB_STARTED_AT": recon_job_started_at, "HOST_RECON_OUTPUT_PATH": f"{recon_path}/output", # Required for nuclei custom-template support: build_nuclei_command # uses this env var to bind-mount mcp/nuclei-templates into the @@ -754,7 +786,7 @@ async def start_partial_recon( volumes={ "/var/run/docker.sock": {"bind": "/var/run/docker.sock", "mode": "rw"}, f"{recon_path}": {"bind": "/app/recon", "mode": "rw"}, - f"{Path(recon_path).parent}/graph_db": {"bind": "/app/graph_db", "mode": "ro"}, + self._sibling_host_path(recon_path, "graph_db"): {"bind": "/app/graph_db", "mode": "ro"}, "/tmp/redamon": {"bind": "/tmp/redamon", "mode": "rw"}, # JS Recon shared volumes with webapp (uploaded files + custom patterns) "redamon_js_recon_uploads": {"bind": "/data/js-recon-uploads", "mode": "ro"}, @@ -1082,7 +1114,7 @@ async def start_gvm_scan( # GVM scan output (read-write, for saving results) f"{gvm_scan_path}/output": {"bind": "/app/gvm_scan/output", "mode": "rw"}, # Mount graph_db module for Neo4j updates - f"{Path(recon_path).parent}/graph_db": {"bind": "/app/graph_db", "mode": "ro"}, + self._sibling_host_path(recon_path, "graph_db"): {"bind": "/app/graph_db", "mode": "ro"}, # Mount gvm_scan source for development (no rebuild needed) f"{gvm_scan_path}": {"bind": "/app/gvm_scan", "mode": "rw"}, }, @@ -1464,7 +1496,7 @@ async def start_github_hunt( # Mount github_secret_hunt source for development (no rebuild needed) f"{github_hunt_path}": {"bind": "/app/github_secret_hunt", "mode": "rw"}, # Mount graph_db module for Neo4j integration - f"{Path(github_hunt_path).parent}/graph_db": {"bind": "/app/graph_db", "mode": "ro"}, + self._sibling_host_path(github_hunt_path, "graph_db"): {"bind": "/app/graph_db", "mode": "ro"}, }, command="python github_secret_hunt/main.py", ) @@ -1838,7 +1870,7 @@ async def start_trufflehog( # Mount trufflehog_scan source for development (no rebuild needed) f"{trufflehog_path}": {"bind": "/app/trufflehog_scan", "mode": "rw"}, # Mount graph_db module for Neo4j integration - f"{Path(trufflehog_path).parent}/graph_db": {"bind": "/app/graph_db", "mode": "ro"}, + self._sibling_host_path(trufflehog_path, "graph_db"): {"bind": "/app/graph_db", "mode": "ro"}, }, command="python trufflehog_scan/main.py", ) diff --git a/recon_orchestrator/models.py b/recon_orchestrator/models.py index 704b38e5..f3fe77ad 100644 --- a/recon_orchestrator/models.py +++ b/recon_orchestrator/models.py @@ -23,6 +23,7 @@ class ReconStartRequest(BaseModel): project_id: str user_id: str webapp_api_url: str + delete_graph: bool = False class ReconState(BaseModel): diff --git a/recon_orchestrator/test_parallel_partial_recon.py b/recon_orchestrator/test_parallel_partial_recon.py index 3122f15f..fc9630fb 100644 --- a/recon_orchestrator/test_parallel_partial_recon.py +++ b/recon_orchestrator/test_parallel_partial_recon.py @@ -14,6 +14,7 @@ import pytest from models import ( + ReconStartRequest, PartialReconState, PartialReconStatus, PartialReconListResponse, @@ -51,6 +52,10 @@ def manager(mock_docker_client): # --------------------------------------------------------------------------- class TestPartialReconModels: + def test_recon_start_request_delete_graph_defaults_false(self): + request = ReconStartRequest(project_id="proj-1", user_id="u1", webapp_api_url="http://localhost") + assert request.delete_graph is False + def test_partial_recon_state_has_run_id(self): state = PartialReconState(project_id="proj-1", run_id="run-abc") assert state.run_id == "run-abc" @@ -95,6 +100,29 @@ def test_partial_recon_state_serialization(self): # --------------------------------------------------------------------------- class TestContainerNaming: + def test_sibling_host_path_preserves_windows_parent(self): + graph_path = ContainerManager._sibling_host_path( + r"D:\Repositories\Gabriel\redamon\recon", + "graph_db", + ) + assert graph_path == r"D:\Repositories\Gabriel\redamon\graph_db" + + def test_sibling_host_path_preserves_posix_parent(self): + graph_path = ContainerManager._sibling_host_path( + "/home/gabriel/redamon/recon", + "graph_db", + ) + assert graph_path == "/home/gabriel/redamon/graph_db" + + def test_format_recon_job_id_uses_prefix_timestamp_and_suffix(self, manager): + started = datetime(2026, 5, 29, 14, 15, 30, tzinfo=timezone.utc) + job_id = manager._format_recon_job_id("full", started, "abcdef1234567890") + assert job_id == "full-20260529T141530Z-abcdef12" + + def test_format_recon_job_started_at_is_utc_iso(self, manager): + started = datetime(2026, 5, 29, 14, 15, 30, tzinfo=timezone.utc) + assert manager._format_recon_job_started_at(started) == "2026-05-29T14:15:30Z" + def test_partial_container_name_includes_run_id(self, manager): name = manager._get_partial_container_name("proj-1", "abcdef12-3456-7890-abcd-ef1234567890") assert "abcdef12" in name @@ -351,6 +379,11 @@ async def test_start_returns_state_with_run_id(self, manager, mock_docker_client assert state.status == PartialReconStatus.RUNNING assert state.container_id == "container-123" assert state.project_id == "proj-1" + env = mock_docker_client.containers.run.call_args.kwargs["environment"] + assert env["RECON_JOB_ID"].startswith("partial-") + assert env["RECON_JOB_ID"].endswith(state.run_id[:8]) + assert env["RECON_JOB_STARTED_AT"].endswith("Z") + assert "RECON_DELETE_GRAPH" not in env @pytest.mark.asyncio async def test_start_stores_in_nested_dict(self, manager, mock_docker_client): @@ -510,6 +543,29 @@ async def test_stop_does_not_cleanup_sub_containers(self, manager, mock_docker_c # --------------------------------------------------------------------------- class TestFullReconMutualExclusion: + @pytest.mark.asyncio + async def test_full_recon_env_includes_delete_graph_and_job_metadata(self, manager, mock_docker_client): + mock_docker_client.images.get.return_value = MagicMock() + mock_container = MagicMock() + mock_container.id = "container-123" + mock_docker_client.containers.run.return_value = mock_container + + with patch('container_manager.Path') as mock_path: + mock_path.return_value = MagicMock() + state = await manager.start_recon( + project_id="proj-1", + user_id="u1", + webapp_api_url="http://localhost", + recon_path="/app/recon", + delete_graph=True, + ) + + assert state.status == ReconStatus.RUNNING + env = mock_docker_client.containers.run.call_args.kwargs["environment"] + assert env["RECON_DELETE_GRAPH"] == "true" + assert env["RECON_JOB_ID"].startswith("full-") + assert env["RECON_JOB_STARTED_AT"].endswith("Z") + @pytest.mark.asyncio async def test_full_recon_blocked_by_partial(self, manager): manager.partial_recon_states["proj-1"] = { diff --git a/tests/test_graph_aux_seen_stamping.py b/tests/test_graph_aux_seen_stamping.py new file mode 100644 index 00000000..03bc5431 --- /dev/null +++ b/tests/test_graph_aux_seen_stamping.py @@ -0,0 +1,184 @@ +import ast +import re +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + +AUX_FILE_LABELS = { + ROOT / "graph_db" / "mixins" / "osint_mixin.py": [ + "IP", + "Port", + "Service", + "Subdomain", + "DNSRecord", + "BaseURL", + "Endpoint", + "Parameter", + "Technology", + "Certificate", + "ExternalDomain", + "ThreatPulse", + "Malware", + "Vulnerability", + ], + ROOT / "graph_db" / "mixins" / "graphql_mixin.py": [ + "BaseURL", + "Endpoint", + "Vulnerability", + ], + ROOT / "graph_db" / "mixins" / "recon" / "takeover_mixin.py": [ + "Domain", + "Subdomain", + "Vulnerability", + ], + ROOT / "graph_db" / "mixins" / "recon" / "vhost_sni_mixin.py": [ + "IP", + "Subdomain", + "BaseURL", + "Endpoint", + "Certificate", + "Vulnerability", + ], + ROOT / "graph_db" / "mixins" / "recon" / "user_input_mixin.py": [ + "UserInput", + "Domain", + "Subdomain", + "IP", + "DNSRecord", + "BaseURL", + "Endpoint", + ], +} + +INTENTIONALLY_ABSENT_LABELS = { + ROOT / "graph_db" / "mixins" / "osint_mixin.py": { + "Technology": "OSINT mixin does not currently create or enrich Technology nodes.", + }, + ROOT / "graph_db" / "mixins" / "recon" / "vhost_sni_mixin.py": { + "Endpoint": "Vhost/SNI does not currently create or enrich Endpoint nodes.", + "Certificate": "Vhost/SNI does not currently create or enrich Certificate nodes.", + }, +} + +GLOBAL_REFERENCE_LABELS = ["CVE", "MitreData", "Capec"] + + +def _node_pattern(keyword, alias, label): + return re.compile( + rf"\b{keyword}\s*\(\s*{re.escape(alias)}\s*:\s*{re.escape(label)}\b" + ) + + +def _alias_has_set(query, alias): + return re.search(rf"(\bSET|,)\s+{re.escape(alias)}\.", query) is not None + + +def _query_constants(tree): + constants = {} + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Constant): + if not isinstance(node.value.value, str): + continue + for target in node.targets: + if isinstance(target, ast.Name): + constants[target.id] = node.value.value + return constants + + +def _query_text(arg, source, constants): + if isinstance(arg, ast.Constant) and isinstance(arg.value, str): + return arg.value + if isinstance(arg, ast.Name): + return constants.get(arg.id) + if isinstance(arg, ast.JoinedStr): + return ast.get_source_segment(source, arg) + return None + + +def _session_run_queries(path): + source = path.read_text(encoding="utf-8") + tree = ast.parse(source) + constants = _query_constants(tree) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if not isinstance(node.func, ast.Attribute) or node.func.attr != "run": + continue + if not node.args: + continue + query = _query_text(node.args[0], source, constants) + if query is None: + continue + call_source = ast.get_source_segment(source, node) or "" + yield query, call_source, node.lineno + + +def _label_aliases(query, labels): + aliases = [] + labels_pattern = "|".join(re.escape(label) for label in labels) + pattern = re.compile( + rf"\b(?:MERGE|MATCH|OPTIONAL MATCH)\s*\(\s*(\w+)\s*:\s*({labels_pattern})\b" + ) + for alias, label in pattern.findall(query): + aliases.append((label, alias)) + return aliases + + +def _is_node_write(query, label, alias): + return ( + _node_pattern("MERGE", alias, label).search(query) is not None + or _alias_has_set(query, alias) + ) + + +def _assert_seen_stamped(query, call_source, path, line, label, alias): + assert ( + f"{alias}.first_seen = coalesce({alias}.first_seen, $recon_job_started_at)" + in query + ), f"{path}:{line} missing first_seen for {label} alias {alias}" + assert ( + f"{alias}.first_seen_job_id = coalesce({alias}.first_seen_job_id, $recon_job_id)" + in query + ), f"{path}:{line} missing first_seen_job_id for {label} alias {alias}" + assert ( + f"{alias}.last_seen = $recon_job_started_at" in query + ), f"{path}:{line} missing last_seen for {label} alias {alias}" + assert ( + f"{alias}.last_seen_job_id = $recon_job_id" in query + ), f"{path}:{line} missing last_seen_job_id for {label} alias {alias}" + assert ( + "_recon_job_params()" in call_source + ), f"{path}:{line} does not pass recon job params for {label} alias {alias}" + + +def test_aux_observable_node_writes_are_seen_stamped_per_block(): + seen_labels_by_path = {path: set() for path in AUX_FILE_LABELS} + for path, labels in AUX_FILE_LABELS.items(): + for query, call_source, line in _session_run_queries(path): + for label, alias in _label_aliases(query, labels): + seen_labels_by_path[path].add(label) + if _is_node_write(query, label, alias): + _assert_seen_stamped(query, call_source, path, line, label, alias) + + for path, labels in AUX_FILE_LABELS.items(): + absent_labels = INTENTIONALLY_ABSENT_LABELS.get(path, {}) + for label in labels: + assert label in seen_labels_by_path[path] or label in absent_labels, ( + f"{path} configured label {label} was not seen and is not documented absent" + ) + for label, reason in absent_labels.items(): + assert label in labels, f"{path} absent label {label} is not configured" + assert label not in seen_labels_by_path[path], ( + f"{path} documents {label} as absent ({reason}) but the label is now present" + ) + + +def test_aux_global_reference_nodes_are_not_seen_stamped(): + for path in AUX_FILE_LABELS: + for query, _call_source, line in _session_run_queries(path): + for label, alias in _label_aliases(query, GLOBAL_REFERENCE_LABELS): + if not _is_node_write(query, label, alias): + continue + assert "first_seen" not in query, f"{path}:{line} stamps {label}" + assert "last_seen" not in query, f"{path}:{line} stamps {label}" diff --git a/tests/test_graph_job_context.py b/tests/test_graph_job_context.py new file mode 100644 index 00000000..514b4635 --- /dev/null +++ b/tests/test_graph_job_context.py @@ -0,0 +1,103 @@ +import importlib.util +import os +import re +import sys +import types +from pathlib import Path +from unittest.mock import MagicMock +from unittest.mock import patch + + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _install_dependency_stubs(): + neo4j_stub = types.ModuleType("neo4j") + neo4j_stub.GraphDatabase = MagicMock() + sys.modules.setdefault("neo4j", neo4j_stub) + + dotenv_stub = types.ModuleType("dotenv") + dotenv_stub.load_dotenv = lambda *args, **kwargs: None + sys.modules.setdefault("dotenv", dotenv_stub) + + graph_db_stub = types.ModuleType("graph_db") + graph_db_stub.__path__ = [str(REPO_ROOT / "graph_db")] + sys.modules.setdefault("graph_db", graph_db_stub) + + mixins_stub = types.ModuleType("graph_db.mixins") + mixins_stub.__path__ = [str(REPO_ROOT / "graph_db" / "mixins")] + sys.modules.setdefault("graph_db.mixins", mixins_stub) + + schema_stub = types.ModuleType("graph_db.schema") + schema_stub.init_schema = MagicMock() + sys.modules.setdefault("graph_db.schema", schema_stub) + + +def _load_base_mixin_module(): + _install_dependency_stubs() + module_name = "graph_db.mixins.base_mixin" + path = REPO_ROOT / "graph_db" / "mixins" / "base_mixin.py" + spec = importlib.util.spec_from_file_location(module_name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +def make_base_without_init(): + BaseMixin = _load_base_mixin_module().BaseMixin + + return BaseMixin.__new__(BaseMixin) + + +def test_graph_job_context_uses_env_values(): + with patch.dict( + os.environ, + { + "RECON_JOB_ID": "full-20260529T141530Z-abcdef12", + "RECON_JOB_STARTED_AT": "2026-05-29T14:15:30Z", + }, + clear=False, + ): + base = make_base_without_init() + base._init_recon_job_context() + + assert base.recon_job_id == "full-20260529T141530Z-abcdef12" + assert base.recon_job_started_at == "2026-05-29T14:15:30Z" + assert base._recon_job_params() == { + "recon_job_id": "full-20260529T141530Z-abcdef12", + "recon_job_started_at": "2026-05-29T14:15:30Z", + } + + +def test_graph_job_context_generates_fallback_values(): + with patch.dict(os.environ, {}, clear=True): + base = make_base_without_init() + base._init_recon_job_context() + + assert base.recon_job_started_at.endswith("Z") + assert re.fullmatch( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z", + base.recon_job_started_at, + ) + assert base.recon_job_id.startswith("adhoc-") + assert re.fullmatch( + r"adhoc-\d{8}T\d{6}Z-[0-9a-f]{8}", + base.recon_job_id, + ) + assert base._recon_job_params() == { + "recon_job_id": base.recon_job_id, + "recon_job_started_at": base.recon_job_started_at, + } + + +def test_node_seen_set_clause_preserves_first_seen_and_updates_last_seen(): + base = make_base_without_init() + clause = base._node_seen_set_clause("d") + + assert clause == ( + "d.first_seen = coalesce(d.first_seen, $recon_job_started_at), " + "d.first_seen_job_id = coalesce(d.first_seen_job_id, $recon_job_id), " + "d.last_seen = $recon_job_started_at, " + "d.last_seen_job_id = $recon_job_id" + ) diff --git a/tests/test_graph_node_seen_stamping.py b/tests/test_graph_node_seen_stamping.py new file mode 100644 index 00000000..bc348195 --- /dev/null +++ b/tests/test_graph_node_seen_stamping.py @@ -0,0 +1,143 @@ +import ast +import re +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + +CORE_FILES = [ + ROOT / "graph_db" / "mixins" / "recon" / "domain_mixin.py", + ROOT / "graph_db" / "mixins" / "recon" / "port_mixin.py", + ROOT / "graph_db" / "mixins" / "recon" / "http_mixin.py", + ROOT / "graph_db" / "mixins" / "recon" / "resource_mixin.py", + ROOT / "graph_db" / "mixins" / "recon" / "js_recon_mixin.py", + ROOT / "graph_db" / "mixins" / "recon" / "vuln_mixin.py", +] + +CORE_LABELS = [ + "Domain", + "Subdomain", + "IP", + "DNSRecord", + "Port", + "Service", + "BaseURL", + "Endpoint", + "Parameter", + "Header", + "Certificate", + "Technology", + "Vulnerability", + "Secret", + "JsReconFinding", +] + +GLOBAL_REFERENCE_LABELS = ["CVE", "MitreData", "Capec"] + + +def _node_pattern(keyword, alias, label): + return re.compile( + rf"\b{keyword}\s*\(\s*{re.escape(alias)}\s*:\s*{re.escape(label)}\b" + ) + + +def _alias_has_set(query, alias): + return re.search(rf"(\bSET|,)\s+{re.escape(alias)}\.", query) is not None + + +def _query_constants(tree): + constants = {} + for node in ast.walk(tree): + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Constant): + if not isinstance(node.value.value, str): + continue + for target in node.targets: + if isinstance(target, ast.Name): + constants[target.id] = node.value.value + return constants + + +def _query_text(arg, source, constants): + if isinstance(arg, ast.Constant) and isinstance(arg.value, str): + return arg.value + if isinstance(arg, ast.Name): + return constants.get(arg.id) + if isinstance(arg, ast.JoinedStr): + return ast.get_source_segment(source, arg) + return None + + +def _session_run_queries(path): + source = path.read_text(encoding="utf-8") + tree = ast.parse(source) + constants = _query_constants(tree) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if not isinstance(node.func, ast.Attribute) or node.func.attr != "run": + continue + if not node.args: + continue + query = _query_text(node.args[0], source, constants) + if query is None: + continue + call_source = ast.get_source_segment(source, node) or "" + yield query, call_source, node.lineno + + +def _label_aliases(query, labels): + aliases = [] + labels_pattern = "|".join(re.escape(label) for label in labels) + pattern = re.compile(rf"\b(?:MERGE|MATCH)\s*\(\s*(\w+)\s*:\s*({labels_pattern})\b") + for alias, label in pattern.findall(query): + aliases.append((label, alias)) + return aliases + + +def _is_node_write(query, label, alias): + return ( + _node_pattern("MERGE", alias, label).search(query) is not None + or _alias_has_set(query, alias) + ) + + +def _assert_seen_stamped(query, call_source, path, line, label, alias): + assert ( + f"{alias}.first_seen = coalesce({alias}.first_seen, $recon_job_started_at)" + in query + ), f"{path}:{line} missing first_seen for {label} alias {alias}" + assert ( + f"{alias}.first_seen_job_id = coalesce({alias}.first_seen_job_id, $recon_job_id)" + in query + ), f"{path}:{line} missing first_seen_job_id for {label} alias {alias}" + assert ( + f"{alias}.last_seen = $recon_job_started_at" in query + ), f"{path}:{line} missing last_seen for {label} alias {alias}" + assert ( + f"{alias}.last_seen_job_id = $recon_job_id" in query + ), f"{path}:{line} missing last_seen_job_id for {label} alias {alias}" + assert ( + "_recon_job_params()" in call_source + ), f"{path}:{line} does not pass recon job params for {label} alias {alias}" + + +def test_core_observable_node_writes_are_seen_stamped_per_block(): + seen_labels = set() + for path in CORE_FILES: + for query, call_source, line in _session_run_queries(path): + for label, alias in _label_aliases(query, CORE_LABELS): + seen_labels.add(label) + if _is_node_write(query, label, alias): + _assert_seen_stamped(query, call_source, path, line, label, alias) + + assert seen_labels == set(CORE_LABELS) + + +def test_global_reference_nodes_are_not_seen_stamped(): + for path in CORE_FILES: + for query, _call_source, line in _session_run_queries(path): + for label, alias in _label_aliases(query, GLOBAL_REFERENCE_LABELS): + if not _is_node_write(query, label, alias): + continue + assert "first_seen" not in query, f"{path}:{line} stamps {label}" + assert "last_seen" not in query, f"{path}:{line} stamps {label}" diff --git a/webapp/src/app/api/recon/[projectId]/start/route.ts b/webapp/src/app/api/recon/[projectId]/start/route.ts index 2537c084..9d47f8a0 100644 --- a/webapp/src/app/api/recon/[projectId]/start/route.ts +++ b/webapp/src/app/api/recon/[projectId]/start/route.ts @@ -11,6 +11,8 @@ interface RouteParams { export async function POST(request: NextRequest, { params }: RouteParams) { try { const { projectId } = await params + const body = await request.json().catch(() => ({})) + const deleteGraph = body?.deleteGraph === true // Verify project exists const project = await prisma.project.findUnique({ @@ -50,6 +52,7 @@ export async function POST(request: NextRequest, { params }: RouteParams) { project_id: projectId, user_id: project.userId, webapp_api_url: WEBAPP_URL, + delete_graph: deleteGraph, }), }) diff --git a/webapp/src/app/graph/components/DataTable/ExpandedRowDetail.tsx b/webapp/src/app/graph/components/DataTable/ExpandedRowDetail.tsx index af218552..c484ed5d 100644 --- a/webapp/src/app/graph/components/DataTable/ExpandedRowDetail.tsx +++ b/webapp/src/app/graph/components/DataTable/ExpandedRowDetail.tsx @@ -12,6 +12,13 @@ interface ExpandedRowDetailProps { row: TableRow } +const PROPERTY_LABELS: Record = { + first_seen: 'First seen', + last_seen: 'Last seen', + first_seen_job_id: 'First seen job', + last_seen_job_id: 'Last seen job', +} + export const ExpandedRowDetail = memo(function ExpandedRowDetail({ row }: ExpandedRowDetailProps) { const properties = Object.entries(row.node.properties) .filter(([key]) => !HIDDEN_KEYS.has(key)) @@ -42,7 +49,7 @@ export const ExpandedRowDetail = memo(function ExpandedRowDetail({ row }: Expand
{properties.map(([key, value]) => (
- {key} + {PROPERTY_LABELS[key] ?? key} {key === 'name' && nodeUrl ? {String(value)} diff --git a/webapp/src/app/graph/components/NodeDetailsTable/NodeDetailsTable.test.tsx b/webapp/src/app/graph/components/NodeDetailsTable/NodeDetailsTable.test.tsx index 32c026a6..6906e34d 100644 --- a/webapp/src/app/graph/components/NodeDetailsTable/NodeDetailsTable.test.tsx +++ b/webapp/src/app/graph/components/NodeDetailsTable/NodeDetailsTable.test.tsx @@ -64,7 +64,15 @@ function makeData(): GraphData { id: 'd1', name: 'example.com', type: 'Domain', - properties: { name: 'example.com', registrar: 'GoDaddy', country: 'US' }, + properties: { + name: 'example.com', + registrar: 'GoDaddy', + country: 'US', + first_seen: '2026-05-29T14:15:30Z', + last_seen: '2026-05-29T14:15:30Z', + first_seen_job_id: 'full-20260529T141530Z-abcdef12', + last_seen_job_id: 'full-20260529T141530Z-abcdef12', + }, }, { id: 'd2', @@ -149,6 +157,24 @@ describe('NodeDetailsTable', () => { expect(headerText).not.toContain('asn') // asn belongs to IP, not Domain }) + test('renders first and last seen metadata columns with readable timestamps', async () => { + const { container } = render( + , + { wrapper: makeWrapper() } + ) + + await waitFor(() => { + const headerText = container.querySelector('thead tr')?.textContent || '' + expect(headerText).toContain('first_seen') + expect(headerText).toContain('last_seen') + expect(headerText).toContain('first_seen_job_id') + expect(headerText).toContain('last_seen_job_id') + }) + + expect(screen.getAllByText('2026-05-29 14:15:30 UTC').length).toBeGreaterThan(0) + expect(screen.getAllByText('full-20260529T141530Z-abcdef12').length).toBeGreaterThan(0) + }) + test('row count badge matches number of nodes of selected type', async () => { render( , @@ -219,12 +245,12 @@ describe('NodeDetailsTable', () => { expect(container.querySelector('thead tr')?.textContent).toContain('registrar') }) - // For Domain: 3 dynamic (registrar/country/city) + 2 fixed-hideable (In/Out) = 5 + // For Domain: 7 dynamic props + 2 fixed-hideable (In/Out) = 9 const colBtn = Array.from(container.querySelectorAll('button')).find(b => b.textContent?.includes('Columns') ) expect(colBtn).toBeDefined() - expect(colBtn!.textContent).toMatch(/5\/5/) + expect(colBtn!.textContent).toMatch(/9\/9/) }) test('"Hide all" button hides every dynamic column; "Show all" restores them', async () => { diff --git a/webapp/src/app/graph/components/NodeDetailsTable/nodeDetailsHelpers.test.ts b/webapp/src/app/graph/components/NodeDetailsTable/nodeDetailsHelpers.test.ts index 9d3a22de..061c3148 100644 --- a/webapp/src/app/graph/components/NodeDetailsTable/nodeDetailsHelpers.test.ts +++ b/webapp/src/app/graph/components/NodeDetailsTable/nodeDetailsHelpers.test.ts @@ -147,6 +147,28 @@ describe('deriveDynamicColumnKeys', () => { expect(a).toEqual(b) expect(a).toEqual(['a', 'b', 'm', 'y', 'z']) }) + + test('prioritizes first and last seen metadata before other keys', () => { + const rows = [ + makeRow(makeNode('Domain', 'd1', { + registrar: 'GoDaddy', + last_seen_job_id: 'full-20260529T141530Z-abcdef12', + first_seen: '2026-05-29T14:15:30Z', + })), + makeRow(makeNode('Domain', 'd2', { + last_seen: '2026-05-29T14:15:30Z', + first_seen_job_id: 'full-20260529T141530Z-abcdef12', + })), + ] + + expect(deriveDynamicColumnKeys(rows)).toEqual([ + 'first_seen', + 'last_seen', + 'first_seen_job_id', + 'last_seen_job_id', + 'registrar', + ]) + }) }) // --------------------------------------------------------------------------- diff --git a/webapp/src/app/graph/components/NodeDetailsTable/nodeDetailsHelpers.ts b/webapp/src/app/graph/components/NodeDetailsTable/nodeDetailsHelpers.ts index da87be9e..f0fa5b5d 100644 --- a/webapp/src/app/graph/components/NodeDetailsTable/nodeDetailsHelpers.ts +++ b/webapp/src/app/graph/components/NodeDetailsTable/nodeDetailsHelpers.ts @@ -10,6 +10,13 @@ export interface GroupedRows { rowsByType: Map } +const PRIORITY_DYNAMIC_KEYS = [ + 'first_seen', + 'last_seen', + 'first_seen_job_id', + 'last_seen_job_id', +] + /** * Partitions a flat list of TableRows by node.type and computes per-type counts * + a sorted type list. Pure — no side effects. @@ -45,7 +52,9 @@ export function deriveDynamicColumnKeys(rows: readonly TableRow[]): string[] { keys.add(k) } } - return Array.from(keys).sort((a, b) => a.localeCompare(b)) + const prioritized = PRIORITY_DYNAMIC_KEYS.filter(key => keys.delete(key)) + const remaining = Array.from(keys).sort((a, b) => a.localeCompare(b)) + return [...prioritized, ...remaining] } /** diff --git a/webapp/src/app/graph/components/ReconConfirmModal/ReconConfirmModal.module.css b/webapp/src/app/graph/components/ReconConfirmModal/ReconConfirmModal.module.css index b9ebbf19..0727b3c7 100644 --- a/webapp/src/app/graph/components/ReconConfirmModal/ReconConfirmModal.module.css +++ b/webapp/src/app/graph/components/ReconConfirmModal/ReconConfirmModal.module.css @@ -33,11 +33,24 @@ border-radius: var(--radius-default); } +.retentionNotice { + display: flex; + gap: var(--space-3); + padding: var(--space-3); + background: var(--bg-tertiary); + border: 1px solid var(--border-default); + border-radius: var(--radius-default); +} + .warningIcon { flex-shrink: 0; color: #ef4444; } +.retentionNotice .warningIcon { + color: var(--text-secondary); +} + .warningContent { flex: 1; } @@ -49,6 +62,10 @@ color: #ef4444; } +.retentionNotice .warningTitle { + color: var(--text-primary); +} + .warningText { margin: 0; font-size: var(--text-xs); @@ -72,6 +89,33 @@ color: var(--text-tertiary); } +.toggleRow { + display: flex; + align-items: center; + gap: var(--space-2); + padding: var(--space-3); + background: var(--bg-secondary); + border: 1px solid var(--border-default); + border-radius: var(--radius-default); + font-size: var(--text-sm); + color: var(--text-primary); + cursor: pointer; +} + +.toggleRow input { + width: 16px; + height: 16px; + margin: 0; + accent-color: var(--accent-primary); + cursor: pointer; +} + +.toggleRow input:disabled, +.toggleRow:has(input:disabled) { + cursor: not-allowed; + opacity: 0.7; +} + .disclaimer { display: flex; gap: var(--space-3); diff --git a/webapp/src/app/graph/components/ReconConfirmModal/ReconConfirmModal.test.tsx b/webapp/src/app/graph/components/ReconConfirmModal/ReconConfirmModal.test.tsx new file mode 100644 index 00000000..e0f05b09 --- /dev/null +++ b/webapp/src/app/graph/components/ReconConfirmModal/ReconConfirmModal.test.tsx @@ -0,0 +1,100 @@ +import { afterEach, describe, test, expect, vi } from 'vitest' +import { cleanup, render, screen, fireEvent } from '@testing-library/react' +import { ReconConfirmModal } from './ReconConfirmModal' + +const defaultProps = { + isOpen: true, + onClose: vi.fn(), + projectName: 'Acme', + targetDomain: 'example.com', + stats: { totalNodes: 3, nodesByType: { Domain: 1, Subdomain: 2 } }, + isLoading: false, +} + +afterEach(cleanup) + +function renderModal(onConfirm = vi.fn(), props = {}) { + const view = render( + + ) + return { onConfirm, ...view } +} + +describe('ReconConfirmModal delete graph toggle', () => { + test('defaults delete graph to off and starts without deletion', () => { + const { onConfirm } = renderModal() + + const checkbox = screen.getByRole('checkbox', { + name: /delete existing graph before starting/i, + }) as HTMLInputElement + expect(checkbox.checked).toBe(false) + expect(screen.getByText(/Existing graph data will be retained/i)).toBeInTheDocument() + + fireEvent.click(screen.getByRole('button', { name: /start recon/i })) + expect(onConfirm).toHaveBeenCalledWith(false) + }) + + test('submits true when delete graph is enabled', () => { + const { onConfirm } = renderModal() + + const checkbox = screen.getByRole('checkbox', { + name: /delete existing graph before starting/i, + }) + fireEvent.click(checkbox) + + expect(screen.getByText(/will delete all existing graph data/i)).toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: /delete & start/i })) + expect(onConfirm).toHaveBeenCalledWith(true) + }) + + test('resets delete graph when the target changes', () => { + const { rerender } = renderModal() + + const checkbox = screen.getByRole('checkbox', { + name: /delete existing graph before starting/i, + }) as HTMLInputElement + fireEvent.click(checkbox) + expect(checkbox.checked).toBe(true) + + rerender( + + ) + + expect(screen.getByRole('checkbox', { + name: /delete existing graph before starting/i, + })).not.toBeChecked() + expect(screen.getByText(/Existing graph data will be retained/i)).toBeInTheDocument() + }) + + test('submits false when existing data is removed before confirming', () => { + const onConfirm = vi.fn() + const { rerender } = renderModal(onConfirm) + + fireEvent.click(screen.getByRole('checkbox', { + name: /delete existing graph before starting/i, + })) + + rerender( + + ) + + expect(screen.queryByRole('checkbox', { + name: /delete existing graph before starting/i, + })).not.toBeInTheDocument() + + fireEvent.click(screen.getByRole('button', { name: /start recon/i })) + expect(onConfirm).toHaveBeenLastCalledWith(false) + }) +}) diff --git a/webapp/src/app/graph/components/ReconConfirmModal/ReconConfirmModal.tsx b/webapp/src/app/graph/components/ReconConfirmModal/ReconConfirmModal.tsx index f9ef162b..ffc2847b 100644 --- a/webapp/src/app/graph/components/ReconConfirmModal/ReconConfirmModal.tsx +++ b/webapp/src/app/graph/components/ReconConfirmModal/ReconConfirmModal.tsx @@ -1,5 +1,6 @@ 'use client' +import { useEffect, useMemo, useState } from 'react' import { AlertTriangle, ShieldAlert, Play, Loader2 } from 'lucide-react' import { Modal } from '@/components/ui' import styles from './ReconConfirmModal.module.css' @@ -12,7 +13,7 @@ interface GraphStats { interface ReconConfirmModalProps { isOpen: boolean onClose: () => void - onConfirm: () => void + onConfirm: (deleteGraph: boolean) => void projectName: string targetDomain: string ipMode?: boolean @@ -32,10 +33,23 @@ export function ReconConfirmModal({ stats, isLoading, }: ReconConfirmModalProps) { + const [deleteGraph, setDeleteGraph] = useState(false) const targetDisplay = ipMode && targetIps?.length ? targetIps.slice(0, 5).join(', ') + (targetIps.length > 5 ? ` (+${targetIps.length - 5} more)` : '') : targetDomain - const hasExistingData = stats && stats.totalNodes > 0 + const existingStats = stats && stats.totalNodes > 0 ? stats : null + const hasExistingData = existingStats !== null + const targetIpsKey = useMemo(() => targetIps?.join('\0') ?? '', [targetIps]) + + useEffect(() => { + if (isOpen || !hasExistingData) { + setDeleteGraph(false) + } + }, [isOpen, projectName, targetDomain, ipMode, targetIpsKey, hasExistingData]) + + const handleConfirm = () => { + onConfirm(deleteGraph && hasExistingData) + } return ( {hasExistingData ? ( -
+

Existing Data Found

- This project has {stats.totalNodes} nodes in the graph database. - Starting a new reconnaissance will delete all existing data and - replace it with fresh scan results. + {deleteGraph ? ( + <> + Starting a new reconnaissance will delete all existing graph data and + replace it with fresh scan results. + + ) : ( + <> + Existing graph data will be retained. New reconnaissance results will be added to + the current {existingStats.totalNodes} graph nodes. + + )}

- {Object.entries(stats.nodesByType).map(([type, count]) => ( + {Object.entries(existingStats.nodesByType).map(([type, count]) => ( {type}: {count} @@ -97,6 +119,18 @@ export function ReconConfirmModal({
)} + {hasExistingData && ( + + )} +
diff --git a/webapp/src/app/graph/page.tsx b/webapp/src/app/graph/page.tsx index 2a35466b..2f35778f 100644 --- a/webapp/src/app/graph/page.tsx +++ b/webapp/src/app/graph/page.tsx @@ -961,13 +961,13 @@ export default function GraphPage() { } }, [searchParams, projectId, router]) - const handleConfirmRecon = useCallback(async () => { + const handleConfirmRecon = useCallback(async (deleteGraph: boolean) => { clearLogs() - const result = await startRecon() + const result = await startRecon({ deleteGraph }) if (result) { setIsReconModalOpen(false) setActiveLogsDrawer('recon') - toast.info('Recon scan started') + toast.info(deleteGraph ? 'Recon scan started; existing graph will be reset' : 'Recon scan started') } }, [startRecon, clearLogs, toast]) diff --git a/webapp/src/app/graph/utils/formatters.ts b/webapp/src/app/graph/utils/formatters.ts index a4e6b559..04ceb02e 100644 --- a/webapp/src/app/graph/utils/formatters.ts +++ b/webapp/src/app/graph/utils/formatters.ts @@ -31,6 +31,34 @@ export function formatNeo4jDateTime(value: unknown): string | null { return null } +/** + * Format ISO UTC timestamp strings to a compact readable display. + */ +export function formatIsoTimestamp(value: unknown): string | null { + if (typeof value !== 'string') { + return null + } + + const match = value.match( + /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2})(?:\.\d+)?Z$/ + ) + if (!match) { + return null + } + + const parsed = new Date(value) + if (Number.isNaN(parsed.getTime())) { + return null + } + + const isoSecond = `${match[1]}T${match[2]}` + if (parsed.toISOString().slice(0, 19) !== isoSecond) { + return null + } + + return `${match[1]} ${match[2]} UTC` +} + /** * Format a property value for display in the drawer */ @@ -41,6 +69,12 @@ export function formatPropertyValue(value: unknown): React.ReactNode { return formattedDate } + const formattedIsoTimestamp = formatIsoTimestamp(value) + + if (formattedIsoTimestamp) { + return formattedIsoTimestamp + } + if ( value === null || value === undefined || diff --git a/webapp/src/hooks/useReconStatus.ts b/webapp/src/hooks/useReconStatus.ts index b20afcbd..253b2ec5 100644 --- a/webapp/src/hooks/useReconStatus.ts +++ b/webapp/src/hooks/useReconStatus.ts @@ -12,12 +12,16 @@ interface UseReconStatusOptions { onError?: (error: string) => void } +export interface StartReconOptions { + deleteGraph?: boolean +} + interface UseReconStatusReturn { state: ReconState | null isLoading: boolean error: string | null refetch: () => Promise - startRecon: () => Promise + startRecon: (options?: StartReconOptions) => Promise stopRecon: () => Promise pauseRecon: () => Promise resumeRecon: () => Promise @@ -85,7 +89,7 @@ export function useReconStatus({ } }, [projectId]) // Only depends on projectId now - const startRecon = useCallback(async (): Promise => { + const startRecon = useCallback(async (options: StartReconOptions = {}): Promise => { if (!projectId) return null setIsLoading(true) @@ -94,6 +98,12 @@ export function useReconStatus({ try { const response = await fetch(`/api/recon/${projectId}/start`, { method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + deleteGraph: options.deleteGraph ?? false, + }), }) if (!response.ok) {