mirror of
https://github.com/reconurge/flowsint.git
synced 2026-11-18 06:35:56 -06:00
refactor(api): usage of graph_repository for all neo4j actions
This commit is contained in:
@@ -22,6 +22,7 @@ from app.api.schemas.investigation import (
|
||||
)
|
||||
from app.api.schemas.sketch import SketchRead
|
||||
from flowsint_core.core.graph_db import neo4j_connection
|
||||
from flowsint_core.core.graph_repository import GraphRepository
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -199,14 +200,11 @@ def delete_investigation(
|
||||
db.query(Analysis).filter(Sketch.investigation_id == investigation_id).all()
|
||||
)
|
||||
|
||||
# Delete all nodes and relationships for each sketch in Neo4j
|
||||
# Delete all nodes and relationships for each sketch in Neo4j using GraphRepository
|
||||
graph_repo = GraphRepository(neo4j_connection)
|
||||
for sketch in sketches:
|
||||
neo4j_query = """
|
||||
MATCH (n {sketch_id: $sketch_id})
|
||||
DETACH DELETE n
|
||||
"""
|
||||
try:
|
||||
neo4j_connection.query(neo4j_query, {"sketch_id": str(sketch.id)})
|
||||
graph_repo.delete_all_sketch_nodes(str(sketch.id))
|
||||
except Exception as e:
|
||||
print(f"Neo4j cleanup error for sketch {sketch.id}: {e}")
|
||||
raise HTTPException(status_code=500, detail="Failed to clean up graph data")
|
||||
|
||||
@@ -17,6 +17,7 @@ from app.api.schemas.sketch import SketchCreate, SketchRead, SketchUpdate
|
||||
from flowsint_core.core.models import Sketch, Profile
|
||||
from uuid import UUID
|
||||
from flowsint_core.core.graph_db import neo4j_connection
|
||||
from flowsint_core.core.graph_repository import GraphRepository
|
||||
from flowsint_core.core.postgre_db import get_db
|
||||
from app.api.deps import get_current_user
|
||||
from flowsint_core.imports import parse_file
|
||||
@@ -145,13 +146,10 @@ def delete_sketch(
|
||||
current_user.id, sketch.investigation_id, actions=["delete"], db=db
|
||||
)
|
||||
|
||||
# Delete all nodes and relationships in Neo4j first
|
||||
neo4j_query = """
|
||||
MATCH (n {sketch_id: $sketch_id})
|
||||
DETACH DELETE n
|
||||
"""
|
||||
# Delete all nodes and relationships in Neo4j first using GraphRepository
|
||||
try:
|
||||
neo4j_connection.query(neo4j_query, {"sketch_id": str(id)})
|
||||
graph_repo = GraphRepository(neo4j_connection)
|
||||
graph_repo.delete_all_sketch_nodes(str(id))
|
||||
except Exception as e:
|
||||
print(f"Neo4j cleanup error: {e}")
|
||||
raise HTTPException(status_code=500, detail="Failed to clean up graph data")
|
||||
@@ -191,25 +189,12 @@ async def get_sketch_nodes(
|
||||
check_investigation_permission(
|
||||
current_user.id, sketch.investigation_id, actions=["read"], db=db
|
||||
)
|
||||
import random
|
||||
# Get all nodes and relationships using GraphRepository
|
||||
graph_repo = GraphRepository(neo4j_connection)
|
||||
graph_data = graph_repo.get_sketch_graph(id, limit=100000)
|
||||
|
||||
nodes_query = """
|
||||
MATCH (n)
|
||||
WHERE n.sketch_id = $sketch_id
|
||||
RETURN elementId(n) as id, labels(n) as labels, properties(n) as data
|
||||
LIMIT 100000
|
||||
"""
|
||||
nodes_result = neo4j_connection.query(nodes_query, parameters={"sketch_id": id})
|
||||
|
||||
node_ids = [record["id"] for record in nodes_result]
|
||||
|
||||
rels_query = """
|
||||
UNWIND $node_ids AS nid
|
||||
MATCH (a)-[r]->(b)
|
||||
WHERE elementId(a) = nid AND elementId(b) IN $node_ids
|
||||
RETURN elementId(r) as id, type(r) as type, elementId(a) as source, elementId(b) as target, properties(r) as data
|
||||
"""
|
||||
rels_result = neo4j_connection.query(rels_query, parameters={"node_ids": node_ids})
|
||||
nodes_result = graph_data["nodes"]
|
||||
rels_result = graph_data["relationships"]
|
||||
|
||||
nodes = [
|
||||
{
|
||||
@@ -330,21 +315,15 @@ def add_edge(
|
||||
current_user.id, sketch.investigation_id, actions=["update"], db=db
|
||||
)
|
||||
|
||||
query = f"""
|
||||
MATCH (a) WHERE elementId(a) = $from_id
|
||||
MATCH (b) WHERE elementId(b) = $to_id
|
||||
MERGE (a)-[r:`{relation.label}` {{sketch_id: $sketch_id}}]->(b)
|
||||
RETURN r
|
||||
"""
|
||||
|
||||
params = {
|
||||
"from_id": relation.source,
|
||||
"to_id": relation.target,
|
||||
"sketch_id": sketch_id,
|
||||
}
|
||||
|
||||
# Create relationship using GraphRepository
|
||||
try:
|
||||
result = neo4j_connection.query(query, params)
|
||||
graph_repo = GraphRepository(neo4j_connection)
|
||||
result = graph_repo.create_relationship_by_element_id(
|
||||
from_element_id=relation.source,
|
||||
to_element_id=relation.target,
|
||||
rel_type=relation.label,
|
||||
sketch_id=sketch_id
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Edge creation error: {e}")
|
||||
raise HTTPException(status_code=500, detail="Failed to create edge")
|
||||
@@ -354,7 +333,7 @@ def add_edge(
|
||||
|
||||
return {
|
||||
"status": "edge added",
|
||||
"edge": result[0]["r"],
|
||||
"edge": result,
|
||||
}
|
||||
|
||||
|
||||
@@ -390,28 +369,21 @@ def edit_node(
|
||||
flattened_data = flatten(node_data)
|
||||
properties.update(flattened_data)
|
||||
|
||||
# Build the SET clause for the Cypher query
|
||||
set_clause = ", ".join(f"n.{key} = ${key}" for key in properties.keys())
|
||||
|
||||
query = f"""
|
||||
MATCH (n)
|
||||
WHERE elementId(n) = $node_id AND n.sketch_id = $sketch_id
|
||||
SET {set_clause}
|
||||
RETURN n as node
|
||||
"""
|
||||
|
||||
params = {"node_id": node_edit.nodeId, "sketch_id": sketch_id, **properties}
|
||||
|
||||
# Update node using GraphRepository
|
||||
try:
|
||||
result = neo4j_connection.query(query, params)
|
||||
graph_repo = GraphRepository(neo4j_connection)
|
||||
updated_node = graph_repo.update_node_by_element_id(
|
||||
element_id=node_edit.nodeId,
|
||||
sketch_id=sketch_id,
|
||||
**properties
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Node update error: {e}")
|
||||
raise HTTPException(status_code=500, detail="Failed to update node")
|
||||
|
||||
if not result:
|
||||
if not updated_node:
|
||||
raise HTTPException(status_code=404, detail="Node not found or not accessible")
|
||||
|
||||
updated_node = result[0]["node"]
|
||||
updated_node["data"] = node_data
|
||||
|
||||
return {
|
||||
@@ -437,23 +409,15 @@ def delete_nodes(
|
||||
current_user.id, sketch.investigation_id, actions=["update"], db=db
|
||||
)
|
||||
|
||||
# Delete nodes and their relationships
|
||||
query = """
|
||||
UNWIND $node_ids AS node_id
|
||||
MATCH (n)
|
||||
WHERE elementId(n) = node_id AND n.sketch_id = $sketch_id
|
||||
DETACH DELETE n
|
||||
"""
|
||||
|
||||
# Delete nodes and their relationships using GraphRepository
|
||||
try:
|
||||
neo4j_connection.query(
|
||||
query, {"node_ids": nodes.nodeIds, "sketch_id": sketch_id}
|
||||
)
|
||||
graph_repo = GraphRepository(neo4j_connection)
|
||||
deleted_count = graph_repo.delete_nodes(nodes.nodeIds, sketch_id)
|
||||
except Exception as e:
|
||||
print(f"Node deletion error: {e}")
|
||||
raise HTTPException(status_code=500, detail="Failed to delete nodes")
|
||||
|
||||
return {"status": "nodes deleted", "count": len(nodes.nodeIds)}
|
||||
return {"status": "nodes deleted", "count": deleted_count}
|
||||
|
||||
|
||||
@router.post("/{sketch_id}/nodes/merge")
|
||||
|
||||
@@ -59,13 +59,13 @@ class GraphRepository:
|
||||
serialized_props["sketch_id"] = sketch_id
|
||||
serialized_props["label"] = serialized_props.get("label", key_value)
|
||||
|
||||
# Build SET clauses
|
||||
set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys()]
|
||||
# Build SET clauses (exclude sketch_id as it's in MERGE)
|
||||
set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys() if prop != "sketch_id"]
|
||||
params = {key_prop: key_value, **serialized_props}
|
||||
|
||||
# Build and execute query
|
||||
# Build and execute query - MERGE on both key_prop AND sketch_id for uniqueness
|
||||
query = f"""
|
||||
MERGE (n:{node_type} {{{key_prop}: ${key_prop}}})
|
||||
MERGE (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})
|
||||
SET {', '.join(set_clauses)}
|
||||
"""
|
||||
|
||||
@@ -111,9 +111,10 @@ class GraphRepository:
|
||||
else:
|
||||
rel_props = "{sketch_id: $sketch_id}"
|
||||
|
||||
# MATCH nodes by both key and sketch_id to ensure we're connecting nodes from the same sketch
|
||||
query = f"""
|
||||
MATCH (from:{from_type} {{{from_key}: $from_value}})
|
||||
MATCH (to:{to_type} {{{to_key}: $to_value}})
|
||||
MATCH (from:{from_type} {{{from_key}: $from_value, sketch_id: $sketch_id}})
|
||||
MATCH (to:{to_type} {{{to_key}: $to_value, sketch_id: $sketch_id}})
|
||||
MERGE (from)-[:{rel_type} {rel_props}]->(to)
|
||||
"""
|
||||
|
||||
@@ -164,11 +165,13 @@ class GraphRepository:
|
||||
serialized_props["sketch_id"] = sketch_id
|
||||
serialized_props["label"] = serialized_props.get("label", key_value)
|
||||
|
||||
set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys()]
|
||||
# Build SET clauses (exclude sketch_id as it's in MERGE)
|
||||
set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys() if prop != "sketch_id"]
|
||||
params = {key_prop: key_value, **serialized_props}
|
||||
|
||||
# MERGE on both key_prop AND sketch_id for uniqueness per sketch
|
||||
query = f"""
|
||||
MERGE (n:{node_type} {{{key_prop}: ${key_prop}}})
|
||||
MERGE (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})
|
||||
SET {', '.join(set_clauses)}
|
||||
"""
|
||||
|
||||
@@ -196,9 +199,10 @@ class GraphRepository:
|
||||
else:
|
||||
rel_props = "{sketch_id: $sketch_id}"
|
||||
|
||||
# MATCH nodes by both key and sketch_id to ensure we're connecting nodes from the same sketch
|
||||
query = f"""
|
||||
MATCH (from:{from_type} {{{from_key}: $from_value}})
|
||||
MATCH (to:{to_type} {{{to_key}: $to_value}})
|
||||
MATCH (from:{from_type} {{{from_key}: $from_value, sketch_id: $sketch_id}})
|
||||
MATCH (to:{to_type} {{{to_key}: $to_value, sketch_id: $sketch_id}})
|
||||
MERGE (from)-[:{rel_type} {rel_props}]->(to)
|
||||
"""
|
||||
|
||||
@@ -239,6 +243,233 @@ class GraphRepository:
|
||||
raise ValueError("Batch size must be at least 1")
|
||||
self._batch_size = size
|
||||
|
||||
def update_node(
|
||||
self,
|
||||
node_type: str,
|
||||
key_prop: str,
|
||||
key_value: str,
|
||||
sketch_id: str,
|
||||
**properties: Any
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Update an existing node's properties.
|
||||
|
||||
Args:
|
||||
node_type: Node label (e.g., "domain", "ip")
|
||||
key_prop: Property name used as unique identifier
|
||||
key_value: Value of the key property
|
||||
sketch_id: Investigation sketch ID
|
||||
**properties: Properties to update
|
||||
|
||||
Returns:
|
||||
Updated node properties or None if not found
|
||||
"""
|
||||
if not self._connection:
|
||||
return None
|
||||
|
||||
# Serialize properties
|
||||
serialized_props = GraphSerializer.serialize_properties(properties)
|
||||
|
||||
# Build SET clauses
|
||||
set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys()]
|
||||
params = {key_prop: key_value, "sketch_id": sketch_id, **serialized_props}
|
||||
|
||||
query = f"""
|
||||
MATCH (n:{node_type} {{{key_prop}: ${key_prop}, sketch_id: $sketch_id}})
|
||||
SET {', '.join(set_clauses)}
|
||||
RETURN properties(n) as node
|
||||
"""
|
||||
|
||||
result = self._connection.query(query, params)
|
||||
return result[0]["node"] if result else None
|
||||
|
||||
def delete_nodes(
|
||||
self,
|
||||
node_ids: List[str],
|
||||
sketch_id: str
|
||||
) -> int:
|
||||
"""
|
||||
Delete nodes by their element IDs.
|
||||
|
||||
Args:
|
||||
node_ids: List of Neo4j element IDs
|
||||
sketch_id: Investigation sketch ID (for safety)
|
||||
|
||||
Returns:
|
||||
Number of nodes deleted
|
||||
"""
|
||||
if not self._connection or not node_ids:
|
||||
return 0
|
||||
|
||||
query = """
|
||||
UNWIND $node_ids AS node_id
|
||||
MATCH (n)
|
||||
WHERE elementId(n) = node_id AND n.sketch_id = $sketch_id
|
||||
DETACH DELETE n
|
||||
RETURN count(n) as deleted_count
|
||||
"""
|
||||
|
||||
result = self._connection.query(
|
||||
query,
|
||||
{"node_ids": node_ids, "sketch_id": sketch_id}
|
||||
)
|
||||
return result[0]["deleted_count"] if result else 0
|
||||
|
||||
def delete_all_sketch_nodes(self, sketch_id: str) -> int:
|
||||
"""
|
||||
Delete all nodes and relationships for a sketch.
|
||||
|
||||
Args:
|
||||
sketch_id: Investigation sketch ID
|
||||
|
||||
Returns:
|
||||
Number of nodes deleted
|
||||
"""
|
||||
if not self._connection:
|
||||
return 0
|
||||
|
||||
query = """
|
||||
MATCH (n {sketch_id: $sketch_id})
|
||||
DETACH DELETE n
|
||||
RETURN count(n) as deleted_count
|
||||
"""
|
||||
|
||||
result = self._connection.query(query, {"sketch_id": sketch_id})
|
||||
return result[0]["deleted_count"] if result else 0
|
||||
|
||||
def get_sketch_graph(
|
||||
self,
|
||||
sketch_id: str,
|
||||
limit: int = 100000
|
||||
) -> Dict[str, List[Dict[str, Any]]]:
|
||||
"""
|
||||
Get all nodes and relationships for a sketch.
|
||||
|
||||
Args:
|
||||
sketch_id: Investigation sketch ID
|
||||
limit: Maximum number of nodes to return
|
||||
|
||||
Returns:
|
||||
Dictionary with 'nodes' and 'relationships' lists
|
||||
"""
|
||||
if not self._connection:
|
||||
return {"nodes": [], "relationships": []}
|
||||
|
||||
# Get all nodes for the sketch
|
||||
nodes_query = """
|
||||
MATCH (n)
|
||||
WHERE n.sketch_id = $sketch_id
|
||||
RETURN elementId(n) as id, labels(n) as labels, properties(n) as data
|
||||
LIMIT $limit
|
||||
"""
|
||||
nodes_result = self._connection.query(
|
||||
nodes_query,
|
||||
{"sketch_id": sketch_id, "limit": limit}
|
||||
)
|
||||
|
||||
if not nodes_result:
|
||||
return {"nodes": [], "relationships": []}
|
||||
|
||||
node_ids = [record["id"] for record in nodes_result]
|
||||
|
||||
# Get all relationships between these nodes
|
||||
rels_query = """
|
||||
UNWIND $node_ids AS nid
|
||||
MATCH (a)-[r]->(b)
|
||||
WHERE elementId(a) = nid AND elementId(b) IN $node_ids
|
||||
RETURN elementId(r) as id, type(r) as type, elementId(a) as source,
|
||||
elementId(b) as target, properties(r) as data
|
||||
"""
|
||||
rels_result = self._connection.query(
|
||||
rels_query,
|
||||
{"node_ids": node_ids}
|
||||
)
|
||||
|
||||
return {
|
||||
"nodes": nodes_result,
|
||||
"relationships": rels_result or []
|
||||
}
|
||||
|
||||
def create_relationship_by_element_id(
|
||||
self,
|
||||
from_element_id: str,
|
||||
to_element_id: str,
|
||||
rel_type: str,
|
||||
sketch_id: str,
|
||||
**properties: Any
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Create a relationship between two nodes using their element IDs.
|
||||
|
||||
Args:
|
||||
from_element_id: Source node element ID
|
||||
to_element_id: Target node element ID
|
||||
rel_type: Relationship type
|
||||
sketch_id: Investigation sketch ID
|
||||
**properties: Additional relationship properties
|
||||
|
||||
Returns:
|
||||
Created relationship properties or None
|
||||
"""
|
||||
if not self._connection:
|
||||
return None
|
||||
|
||||
serialized_props = GraphSerializer.serialize_properties(properties)
|
||||
serialized_props["sketch_id"] = sketch_id
|
||||
|
||||
props_str = ", ".join([f"{k}: ${k}" for k in serialized_props.keys()])
|
||||
rel_props = f"{{{props_str}}}"
|
||||
|
||||
query = f"""
|
||||
MATCH (a) WHERE elementId(a) = $from_id
|
||||
MATCH (b) WHERE elementId(b) = $to_id
|
||||
MERGE (a)-[r:`{rel_type}` {rel_props}]->(b)
|
||||
RETURN properties(r) as rel
|
||||
"""
|
||||
|
||||
params = {
|
||||
"from_id": from_element_id,
|
||||
"to_id": to_element_id,
|
||||
**serialized_props
|
||||
}
|
||||
|
||||
result = self._connection.query(query, params)
|
||||
return result[0]["rel"] if result else None
|
||||
|
||||
def update_node_by_element_id(
|
||||
self,
|
||||
element_id: str,
|
||||
sketch_id: str,
|
||||
**properties: Any
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Update a node by its element ID.
|
||||
|
||||
Args:
|
||||
element_id: Neo4j element ID
|
||||
sketch_id: Investigation sketch ID (for safety)
|
||||
**properties: Properties to update
|
||||
|
||||
Returns:
|
||||
Updated node properties or None if not found
|
||||
"""
|
||||
if not self._connection:
|
||||
return None
|
||||
|
||||
serialized_props = GraphSerializer.serialize_properties(properties)
|
||||
set_clauses = [f"n.{prop} = ${prop}" for prop in serialized_props.keys()]
|
||||
|
||||
query = f"""
|
||||
MATCH (n)
|
||||
WHERE elementId(n) = $element_id AND n.sketch_id = $sketch_id
|
||||
SET {', '.join(set_clauses)}
|
||||
RETURN properties(n) as node
|
||||
"""
|
||||
|
||||
params = {"element_id": element_id, "sketch_id": sketch_id, **serialized_props}
|
||||
result = self._connection.query(query, params)
|
||||
return result[0]["node"] if result else None
|
||||
|
||||
def query(self, cypher: str, parameters: Dict[str, Any] = None) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Execute a custom Cypher query.
|
||||
|
||||
Reference in New Issue
Block a user