diff --git a/.env.example b/.env.example index 77dc153..a66fd59 100644 --- a/.env.example +++ b/.env.example @@ -1,10 +1,9 @@ NODE_ENV=production NEXT_PUBLIC_AUTH_REDIRECT=http://app.flowsint.localhost/auth/callback -HIBP_API_KEY=70b78a3256c84d09b79cd4953d77bdf3 +HIBP_API_KEY= NEXT_PUBLIC_FLOWSINT_API=https://api.flowsint.localhost/api NEXT_PUBLIC_DOCKER_FLOWSINT_API=http://flowsint-api:5000/api -NEXT_PUBLIC_SUPABASE_ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InR5bGhuc2F5eXRhb2FhaXFzZ2RwIiwicm9sZSI6ImFub24iLCJpYXQiOjE3MzgzMzY3MzAsImV4cCI6MjA1MzkxMjczMH0.iVcpz8RpOgzVSamp_tNQQmjdLL9_Olx6m4LfsLVw1bg -AUTH_SECRET="superscretchangeitplz" # Added by `npx auth`. Read more: https://cli.authjs.dev +AUTH_SECRET="superscretchangeitplz" AUTH_TRUST_HOST=true NEO4J_URI_BOLT=bolt://neo4j:7687 NEO4J_URI_WEB=https://neo4j.flowsint.localhost diff --git a/flowsint-api/alembic/versions/661ff8ef4425_rename_transforms_to_flows.py b/flowsint-api/alembic/versions/661ff8ef4425_rename_transforms_to_flows.py new file mode 100644 index 0000000..60f2fc1 --- /dev/null +++ b/flowsint-api/alembic/versions/661ff8ef4425_rename_transforms_to_flows.py @@ -0,0 +1,36 @@ +"""rename_transforms_to_flows + +Revision ID: 661ff8ef4425 +Revises: 9a3b9a199aa8 +Create Date: 2025-08-15 16:16:12.792775 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '661ff8ef4425' +down_revision: Union[str, None] = '9a3b9a199aa8' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # Rename the table from 'transforms' to 'flows' + op.rename_table('transforms', 'flows') + + # Rename the column from 'transform_schema' to 'flow_schema' + op.alter_column('flows', 'transform_schema', new_column_name='flow_schema') + + +def downgrade() -> None: + """Downgrade schema.""" + # Rename the column back from 'flow_schema' to 'transform_schema' + op.alter_column('flows', 'flow_schema', new_column_name='transform_schema') + + # Rename the table back from 'flows' to 'transforms' + op.rename_table('flows', 'transforms') diff --git a/flowsint-api/app/api/routes/flows.py b/flowsint-api/app/api/routes/flows.py new file mode 100644 index 0000000..2c5bed9 --- /dev/null +++ b/flowsint-api/app/api/routes/flows.py @@ -0,0 +1,548 @@ +from uuid import UUID, uuid4 +from fastapi import APIRouter, HTTPException, Depends, status, Query +from typing import Dict, List, Any, Optional +from pydantic import BaseModel +from datetime import datetime +from flowsint_core.utils import extract_input_schema_flow +from flowsint_core.core.registry import TransformRegistry +from flowsint_core.core.celery import celery +from flowsint_types import Domain, Phrase, Ip, SocialProfile, Organization, Email +from flowsint_core.core.types import Node, Edge, FlowStep, FlowBranch +from sqlalchemy.orm import Session +from flowsint_core.core.postgre_db import get_db +from app.models.models import Flow, Profile +from app.api.deps import get_current_user +from app.api.schemas.flow import FlowRead, FlowCreate, FlowUpdate +from flowsint_types import ( + ASN, + CIDR, + CryptoWallet, + CryptoWalletTransaction, + CryptoNFT, + Website, + Individual, +) + + +class FlowComputationRequest(BaseModel): + nodes: List[Node] + edges: List[Edge] + inputType: Optional[str] = None + + +class FlowComputationResponse(BaseModel): + flowBranches: List[FlowBranch] + initialData: Any + + +class StepSimulationRequest(BaseModel): + flowBranches: List[FlowBranch] + currentStepIndex: int + + +class launchFlowPayload(BaseModel): + values: List[str] + sketch_id: str + + +router = APIRouter() + + +# Get the list of all flows +@router.get("/", response_model=List[FlowRead]) +def get_flows( + category: Optional[str] = Query(None), + db: Session = Depends(get_db), + current_user: Profile = Depends(get_current_user), +): + query = db.query(Flow) + + if category is not None and category != "undefined": + # Case-insensitive filtering by checking if any category matches (case-insensitive) + flows = query.all() + return [ + flow + for flow in flows + if any(cat.lower() == category.lower() for cat in flow.category) + ] + + return query.order_by(Flow.last_updated_at.desc()).all() + + +# Returns the "raw_materials" for the flow editor +@router.get("/raw_materials") +async def get_material_list(): + scanners = TransformRegistry.list_by_categories() + scanner_categories = { + category: [ + { + "class_name": scanner.get("class_name"), + "category": scanner.get("category"), + "name": scanner.get("name"), + "module": scanner.get("module"), + "documentation": scanner.get("documentation"), + "description": scanner.get("description"), + "inputs": scanner.get("inputs"), + "outputs": scanner.get("outputs"), + "type": "scanner", + "params": scanner.get("params"), + "params_schema": scanner.get("params_schema"), + "required_params": scanner.get("required_params"), + "icon": scanner.get("icon"), + } + for scanner in scanner_list + ] + for category, scanner_list in scanners.items() + } + + object_inputs = [ + extract_input_schema_flow(Phrase), + extract_input_schema_flow(Organization), + extract_input_schema_flow(Individual), + extract_input_schema_flow(Domain), + extract_input_schema_flow(Website), + extract_input_schema_flow(Ip), + extract_input_schema_flow(ASN), + extract_input_schema_flow(CIDR), + extract_input_schema_flow(SocialProfile), + extract_input_schema_flow(Email), + extract_input_schema_flow(CryptoWallet), + extract_input_schema_flow(CryptoWalletTransaction), + extract_input_schema_flow(CryptoNFT), + ] + + # Put types first, then add all scanner categories + flattened_scanners = {"types": object_inputs} + flattened_scanners.update(scanner_categories) + + return {"items": flattened_scanners} + + +# Returns the "raw_materials" for the flow editor +@router.get("/input_type/{input_type}") +async def get_material_list(input_type: str): + transforms = TransformRegistry.list_by_input_type(input_type) + return {"items": transforms} + + +# Create a new flow +@router.post("/create", response_model=FlowRead, status_code=status.HTTP_201_CREATED) +def create_flow( + payload: FlowCreate, + db: Session = Depends(get_db), + current_user: Profile = Depends(get_current_user), +): + + new_flow = Flow( + id=uuid4(), + name=payload.name, + description=payload.description, + category=payload.category, + flow_schema=payload.flow_schema, + created_at=datetime.utcnow(), + last_updated_at=datetime.utcnow(), + ) + db.add(new_flow) + db.commit() + db.refresh(new_flow) + return new_flow + + +# Get a flow by ID +@router.get("/{flow_id}", response_model=FlowRead) +def get_flow_by_id( + flow_id: UUID, + db: Session = Depends(get_db), + current_user: Profile = Depends(get_current_user), +): + flow = db.query(Flow).filter(Flow.id == flow_id).first() + if not flow: + raise HTTPException(status_code=404, detail="flow not found") + return flow + + +# Update a flow by ID +@router.put("/{flow_id}", response_model=FlowRead) +def update_flow( + flow_id: UUID, + payload: FlowUpdate, + db: Session = Depends(get_db), + current_user: Profile = Depends(get_current_user), +): + flow = db.query(Flow).filter(Flow.id == flow_id).first() + if not flow: + raise HTTPException(status_code=404, detail="flow not found") + update_data = payload.dict(exclude_unset=True) + for key, value in update_data.items(): + print(f"only update {key}") + if key == "category": + if "SocialProfile" in value: + value.append("Username") + setattr(flow, key, value) + + flow.last_updated_at = datetime.utcnow() + + db.commit() + db.refresh(flow) + return flow + + +# Delete a flow by ID +@router.delete("/{flow_id}", status_code=status.HTTP_204_NO_CONTENT) +def delete_flow( + flow_id: UUID, + db: Session = Depends(get_db), + current_user: Profile = Depends(get_current_user), +): + flow = db.query(Flow).filter(Flow.id == flow_id).first() + if not flow: + raise HTTPException(status_code=404, detail="flow not found") + db.delete(flow) + db.commit() + return None + + +@router.post("/{flow_id}/launch") +async def launch_flow( + flow_id: str, + payload: launchFlowPayload, + db: Session = Depends(get_db), + current_user: Profile = Depends(get_current_user), +): + try: + flow = db.query(Flow).filter(Flow.id == flow_id).first() + if flow is None: + raise HTTPException(status_code=404, detail="flow not found") + nodes = [Node(**node) for node in flow.flow_schema["nodes"]] + edges = [Edge(**edge) for edge in flow.flow_schema["edges"]] + flow_branches = compute_flow_branches(payload.values, nodes, edges) + serializable_branches = [branch.model_dump() for branch in flow_branches] + task = celery.send_task( + "run_flow", + args=[ + serializable_branches, + payload.values, + payload.sketch_id, + str(current_user.id), + ], + ) + return {"id": task.id} + + except Exception as e: + print(e) + raise HTTPException(status_code=404, detail="flow not found") + + +@router.post("/{flow_id}/compute", response_model=FlowComputationResponse) +def compute_flows( + request: FlowComputationRequest, current_user: Profile = Depends(get_current_user) +): + initial_data = generate_sample_data(request.inputType or "string") + flow_branches = compute_flow_branches(initial_data, request.nodes, request.edges) + return FlowComputationResponse(flowBranches=flow_branches, initialData=initial_data) + + +def generate_sample_data(type_str: str) -> Any: + type_str = type_str.lower() if type_str else "string" + if type_str == "string": + return "sample_text" + elif type_str == "number": + return 42 + elif type_str == "boolean": + return True + elif type_str == "array": + return [1, 2, 3] + elif type_str == "object": + return {"key": "value"} + elif type_str == "url": + return "https://example.com" + elif type_str == "email": + return "user@example.com" + elif type_str == "domain": + return "example.com" + elif type_str == "ip": + return "192.168.1.1" + else: + return f"sample_{type_str}" + + +def compute_flow_branches( + initial_value: Any, nodes: List[Node], edges: List[Edge] +) -> List[FlowBranch]: + """Computes flow branches based on nodes and edges with proper DFS traversal""" + # Find input nodes (starting points) + input_nodes = [node for node in nodes if node.data.get("type") == "type"] + + if not input_nodes: + return [ + FlowBranch( + id="error", + name="Error", + steps=[ + FlowStep( + nodeId="error", + inputs={}, + type="error", + outputs={}, + status="error", + branchId="error", + depth=0, + ) + ], + ) + ] + + node_map = {node.id: node for node in nodes} + branches = [] + branch_counter = 0 + # Track scanner outputs across all branches + scanner_outputs = {} + + def calculate_path_length(start_node: str, visited: set = None) -> int: + """Calculate the shortest possible path length from a node to any leaf""" + if visited is None: + visited = set() + + if start_node in visited: + return float("inf") + + visited.add(start_node) + out_edges = [edge for edge in edges if edge.source == start_node] + + if not out_edges: + return 1 + + min_length = float("inf") + for edge in out_edges: + length = calculate_path_length(edge.target, visited.copy()) + min_length = min(min_length, length) + + return 1 + min_length + + def get_outgoing_edges(node_id: str) -> List[Edge]: + """Get outgoing edges sorted by the shortest possible path length""" + out_edges = [edge for edge in edges if edge.source == node_id] + # Sort edges by the length of the shortest possible path from their target + return sorted(out_edges, key=lambda e: calculate_path_length(e.target)) + + def create_step( + node_id: str, + branch_id: str, + depth: int, + input_data: Dict[str, Any], + is_input_node: bool, + outputs: Dict[str, Any], + node_params: Optional[Dict[str, Any]] = None, + ) -> FlowStep: + return FlowStep( + nodeId=node_id, + params=node_params, + inputs={} if is_input_node else input_data, + outputs=outputs, + type="type" if is_input_node else "scanner", + status="pending", + branchId=branch_id, + depth=depth, + ) + + def explore_branch( + current_node_id: str, + branch_id: str, + branch_name: str, + depth: int, + input_data: Dict[str, Any], + path: List[str], + branch_visited: set, + steps: List[FlowStep], + parent_outputs: Dict[str, Any] = None, + ) -> None: + nonlocal branch_counter + + # Skip if node is already in current path (cycle detection) + if current_node_id in path: + return + + current_node = node_map.get(current_node_id) + if not current_node: + return + + # Process node outputs + is_input_node = current_node.data.get("type") == "type" + if is_input_node: + outputs_array = current_node.data["outputs"].get("properties", []) + first_output_name = ( + outputs_array[0].get("name", "output") if outputs_array else "output" + ) + current_outputs = {first_output_name: initial_value} + else: + # Check if we already have outputs for this scanner + if current_node_id in scanner_outputs: + current_outputs = scanner_outputs[current_node_id] + else: + current_outputs = process_node_data(current_node, input_data) + # Store the outputs for future use + scanner_outputs[current_node_id] = current_outputs + + # Extract node parameters + node_params = current_node.data.get("params", {}) + + # Create and add current step + current_step = create_step( + current_node_id, + branch_id, + depth, + input_data, + is_input_node, + current_outputs, + node_params, + ) + steps.append(current_step) + path.append(current_node_id) + branch_visited.add(current_node_id) + + # Get all outgoing edges sorted by path length + out_edges = get_outgoing_edges(current_node_id) + + if not out_edges: + # Leaf node reached, save the branch + branches.append(FlowBranch(id=branch_id, name=branch_name, steps=steps[:])) + else: + # Process each outgoing edge in order of shortest path + for i, edge in enumerate(out_edges): + if edge.target in path: # Skip if would create cycle + continue + + # Prepare next node's input + output_key = edge.sourceHandle + if not output_key and current_outputs: + output_key = list(current_outputs.keys())[0] + + output_value = current_outputs.get(output_key) if output_key else None + if output_value is None and parent_outputs: + output_value = ( + parent_outputs.get(output_key) if output_key else None + ) + + next_input = {edge.targetHandle or "input": output_value} + + if i == 0: + # Continue in same branch (will be shortest path) + explore_branch( + edge.target, + branch_id, + branch_name, + depth + 1, + next_input, + path, + branch_visited, + steps, + current_outputs, + ) + else: + # Create new branch starting from current node + branch_counter += 1 + new_branch_id = f"{branch_id}-{branch_counter}" + new_branch_name = f"{branch_name} (Branch {branch_counter})" + new_steps = steps[: len(steps)] # Copy steps up to current node + new_branch_visited = ( + branch_visited.copy() + ) # Create new visited set for the branch + explore_branch( + edge.target, + new_branch_id, + new_branch_name, + depth + 1, + next_input, + path[:], # Create new path copy for branch + new_branch_visited, + new_steps, + current_outputs, + ) + + # Backtrack: remove current node from path and remove its step + path.pop() + steps.pop() + + # Start exploration from each input node + for index, input_node in enumerate(input_nodes): + branch_id = f"branch-{index}" + branch_name = f"Flow {index + 1}" if len(input_nodes) > 1 else "Main Flow" + explore_branch( + input_node.id, + branch_id, + branch_name, + 0, + {}, + [], # Use list for path to maintain order + set(), # Use set for visited to check membership + [], + None, + ) + + # Sort branches by length (number of steps) + branches.sort(key=lambda branch: len(branch.steps)) + return branches + + +def process_node_data(node: Node, inputs: Dict[str, Any]) -> Dict[str, Any]: + """Traite les données de nœud en fonction du type de nœud et des entrées""" + outputs = {} + output_types = node.data["outputs"].get("properties", []) + + for output in output_types: + output_name = output.get("name", "output") + class_name = node.data.get("class_name", "") + # For simulation purposes, we'll return a placeholder value based on the scanner type + if class_name in ["ReverseResolveScanner", "ResolveScanner"]: + # IP/Domain resolution scanners + outputs[output_name] = ( + "192.168.1.1" if "ip" in output_name.lower() else "example.com" + ) + elif class_name == "SubdomainScanner": + # Subdomain scanner + outputs[output_name] = f"sub.{inputs.get('input', 'example.com')}" + + elif class_name == "WhoisScanner": + # WHOIS scanner + outputs[output_name] = { + "domain": inputs.get("input", "example.com"), + "registrar": "Example Registrar", + "creation_date": "2020-01-01", + } + + elif class_name == "GeolocationScanner": + # Geolocation scanner + outputs[output_name] = { + "country": "France", + "city": "Paris", + "coordinates": {"lat": 48.8566, "lon": 2.3522}, + } + + elif class_name == "MaigretScanner": + # Social media scanner + outputs[output_name] = { + "username": inputs.get("input", "user123"), + "platforms": ["twitter", "github", "linkedin"], + } + + elif class_name == "HoleheScanner": + # Email verification scanner + outputs[output_name] = { + "email": inputs.get("input", "user@example.com"), + "exists": True, + "platforms": ["gmail", "github"], + } + + elif class_name == "SireneScanner": + # Organization scanner + outputs[output_name] = { + "name": inputs.get("input", "Example Corp"), + "siret": "12345678901234", + "address": "1 Example Street", + } + + else: + # For unknown scanners, pass through the input + outputs[output_name] = inputs.get("input") or f"flowed_{output_name}" + + return outputs diff --git a/flowsint-api/app/api/routes/investigations.py b/flowsint-api/app/api/routes/investigations.py index 8661e48..4bc0add 100644 --- a/flowsint-api/app/api/routes/investigations.py +++ b/flowsint-api/app/api/routes/investigations.py @@ -23,7 +23,10 @@ def get_investigations( db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user) ): investigations = ( - db.query(Investigation).filter(Investigation.owner_id == current_user.id).all() + db.query(Investigation) + .options(selectinload(Investigation.sketches), selectinload(Investigation.analyses), selectinload(Investigation.owner)) + .filter(Investigation.owner_id == current_user.id) + .all() ) return investigations @@ -61,7 +64,7 @@ def get_investigation_by_id( ): investigation = ( db.query(Investigation) - .options(selectinload(Investigation.sketches)) + .options(selectinload(Investigation.sketches), selectinload(Investigation.analyses), selectinload(Investigation.owner)) .filter(Investigation.id == investigation_id) .filter(Investigation.owner_id == current_user.id) .first() diff --git a/flowsint-api/app/api/routes/transforms.py b/flowsint-api/app/api/routes/transforms.py index 475e3fa..d44d3ca 100644 --- a/flowsint-api/app/api/routes/transforms.py +++ b/flowsint-api/app/api/routes/transforms.py @@ -1,27 +1,11 @@ -from uuid import UUID, uuid4 -from fastapi import APIRouter, HTTPException, Depends, status, Query -from typing import Dict, List, Any, Optional +from fastapi import APIRouter, HTTPException, Depends, Query +from typing import List, Any, Optional from pydantic import BaseModel -from datetime import datetime -from flowsint_core.utils import extract_input_schema_transform -from flowsint_core.core.registry import ScannerRegistry +from flowsint_core.core.registry import TransformRegistry from flowsint_core.core.celery import celery -from flowsint_types import Domain, Phrase, Ip, SocialProfile, Organization, Email -from flowsint_core.core.types import Node, Edge, FlowStep, FlowBranch -from sqlalchemy.orm import Session -from flowsint_core.core.postgre_db import get_db -from app.models.models import Transform, Profile +from flowsint_core.core.types import Node, Edge, FlowBranch +from app.models.models import Profile from app.api.deps import get_current_user -from app.api.schemas.transform import TransformRead, TransformCreate, TransformUpdate -from flowsint_types import ( - ASN, - CIDR, - CryptoWallet, - CryptoWalletTransaction, - CryptoNFT, - Website, - Individual, -) class FlowComputationRequest(BaseModel): @@ -31,16 +15,16 @@ class FlowComputationRequest(BaseModel): class FlowComputationResponse(BaseModel): - transformBranches: List[FlowBranch] + flowBranches: List[FlowBranch] initialData: Any class StepSimulationRequest(BaseModel): - transformBranches: List[FlowBranch] + flowBranches: List[FlowBranch] currentStepIndex: int -class LaunchTransformPayload(BaseModel): +class launchTransformPayload(BaseModel): values: List[str] sketch_id: str @@ -49,180 +33,29 @@ router = APIRouter() # Get the list of all transforms -@router.get("", response_model=List[TransformRead]) +@router.get("/") def get_transforms( category: Optional[str] = Query(None), - db: Session = Depends(get_db), - current_user: Profile = Depends(get_current_user), + # current_user: Profile = Depends(get_current_user), ): - query = db.query(Transform) - if category is not None and category != "undefined": - # Case-insensitive filtering by checking if any category matches (case-insensitive) - transforms = query.all() - return [ - transform - for transform in transforms - if any(cat.lower() == category.lower() for cat in transform.category) - ] - - return query.order_by(Transform.last_updated_at.desc()).all() + transforms = TransformRegistry.list_by_input_type(category) + else: + transforms = TransformRegistry.list() + return transforms -# Returns the "raw_materials" for the transform editor -@router.get("/raw_materials") -async def get_material_list(): - scanners = ScannerRegistry.list_by_categories() - scanner_categories = { - category: [ - { - "class_name": scanner.get("class_name"), - "category": scanner.get("category"), - "name": scanner.get("name"), - "module": scanner.get("module"), - "documentation": scanner.get("documentation"), - "description": scanner.get("description"), - "inputs": scanner.get("inputs"), - "outputs": scanner.get("outputs"), - "type": "scanner", - "params": scanner.get("params"), - "params_schema": scanner.get("params_schema"), - "required_params": scanner.get("required_params"), - "icon": scanner.get("icon"), - } - for scanner in scanner_list - ] - for category, scanner_list in scanners.items() - } - - object_inputs = [ - extract_input_schema_transform(Phrase), - extract_input_schema_transform(Organization), - extract_input_schema_transform(Individual), - extract_input_schema_transform(Domain), - extract_input_schema_transform(Website), - extract_input_schema_transform(Ip), - extract_input_schema_transform(ASN), - extract_input_schema_transform(CIDR), - extract_input_schema_transform(SocialProfile), - extract_input_schema_transform(Email), - extract_input_schema_transform(CryptoWallet), - extract_input_schema_transform(CryptoWalletTransaction), - extract_input_schema_transform(CryptoNFT), - ] - - # Put types first, then add all scanner categories - flattened_scanners = {"types": object_inputs} - flattened_scanners.update(scanner_categories) - - return {"items": flattened_scanners} - - -# Returns the "raw_materials" for the transform editor -@router.get("/input_type/{input_type}") -async def get_material_list(input_type: str): - scanners = ScannerRegistry.list_by_input_type(input_type) - return {"items": scanners} - - -# Create a new transform -@router.post( - "/create", response_model=TransformRead, status_code=status.HTTP_201_CREATED -) -def create_transform( - payload: TransformCreate, - db: Session = Depends(get_db), - current_user: Profile = Depends(get_current_user), -): - - new_transform = Transform( - id=uuid4(), - name=payload.name, - description=payload.description, - category=payload.category, - transform_schema=payload.transform_schema, - created_at=datetime.utcnow(), - last_updated_at=datetime.utcnow(), - ) - db.add(new_transform) - db.commit() - db.refresh(new_transform) - return new_transform - - -# Get a transform by ID -@router.get("/{transform_id}", response_model=TransformRead) -def get_transform_by_id( - transform_id: UUID, - db: Session = Depends(get_db), - current_user: Profile = Depends(get_current_user), -): - transform = db.query(Transform).filter(Transform.id == transform_id).first() - if not transform: - raise HTTPException(status_code=404, detail="Transform not found") - return transform - - -# Update a transform by ID -@router.put("/{transform_id}", response_model=TransformRead) -def update_transform( - transform_id: UUID, - payload: TransformUpdate, - db: Session = Depends(get_db), - current_user: Profile = Depends(get_current_user), -): - transform = db.query(Transform).filter(Transform.id == transform_id).first() - if not transform: - raise HTTPException(status_code=404, detail="Transform not found") - update_data = payload.dict(exclude_unset=True) - for key, value in update_data.items(): - print(f"only update {key}") - if key == "category": - if "SocialProfile" in value: - value.append("Username") - setattr(transform, key, value) - - transform.last_updated_at = datetime.utcnow() - - db.commit() - db.refresh(transform) - return transform - - -# Delete a transform by ID -@router.delete("/{transform_id}", status_code=status.HTTP_204_NO_CONTENT) -def delete_transform( - transform_id: UUID, - db: Session = Depends(get_db), - current_user: Profile = Depends(get_current_user), -): - transform = db.query(Transform).filter(Transform.id == transform_id).first() - if not transform: - raise HTTPException(status_code=404, detail="Transform not found") - db.delete(transform) - db.commit() - return None - - -@router.post("/{transform_id}/launch") +@router.post("/{transform_name}/launch") async def launch_transform( - transform_id: str, - payload: LaunchTransformPayload, - db: Session = Depends(get_db), + transform_name: str, + payload: launchTransformPayload, current_user: Profile = Depends(get_current_user), ): try: - transform = db.query(Transform).filter(Transform.id == transform_id).first() - if transform is None: - raise HTTPException(status_code=404, detail="Transform not found") - nodes = [Node(**node) for node in transform.transform_schema["nodes"]] - edges = [Edge(**edge) for edge in transform.transform_schema["edges"]] - transform_branches = compute_transform_branches(payload.values, nodes, edges) - serializable_branches = [branch.model_dump() for branch in transform_branches] task = celery.send_task( "run_transform", args=[ - serializable_branches, + transform_name, payload.values, payload.sketch_id, str(current_user.id), @@ -233,322 +66,3 @@ async def launch_transform( except Exception as e: print(e) raise HTTPException(status_code=404, detail="Transform not found") - - -@router.post("/{transform_id}/compute", response_model=FlowComputationResponse) -def compute_transforms( - request: FlowComputationRequest, current_user: Profile = Depends(get_current_user) -): - initial_data = generate_sample_data(request.inputType or "string") - transform_branches = compute_transform_branches( - initial_data, request.nodes, request.edges - ) - return FlowComputationResponse( - transformBranches=transform_branches, initialData=initial_data - ) - - -def generate_sample_data(type_str: str) -> Any: - type_str = type_str.lower() if type_str else "string" - if type_str == "string": - return "sample_text" - elif type_str == "number": - return 42 - elif type_str == "boolean": - return True - elif type_str == "array": - return [1, 2, 3] - elif type_str == "object": - return {"key": "value"} - elif type_str == "url": - return "https://example.com" - elif type_str == "email": - return "user@example.com" - elif type_str == "domain": - return "example.com" - elif type_str == "ip": - return "192.168.1.1" - else: - return f"sample_{type_str}" - - -def compute_transform_branches( - initial_value: Any, nodes: List[Node], edges: List[Edge] -) -> List[FlowBranch]: - """Computes flow branches based on nodes and edges with proper DFS traversal""" - # Find input nodes (starting points) - input_nodes = [node for node in nodes if node.data.get("type") == "type"] - - if not input_nodes: - return [ - FlowBranch( - id="error", - name="Error", - steps=[ - FlowStep( - nodeId="error", - inputs={}, - type="error", - outputs={}, - status="error", - branchId="error", - depth=0, - ) - ], - ) - ] - - node_map = {node.id: node for node in nodes} - branches = [] - branch_counter = 0 - # Track scanner outputs across all branches - scanner_outputs = {} - - def calculate_path_length(start_node: str, visited: set = None) -> int: - """Calculate the shortest possible path length from a node to any leaf""" - if visited is None: - visited = set() - - if start_node in visited: - return float("inf") - - visited.add(start_node) - out_edges = [edge for edge in edges if edge.source == start_node] - - if not out_edges: - return 1 - - min_length = float("inf") - for edge in out_edges: - length = calculate_path_length(edge.target, visited.copy()) - min_length = min(min_length, length) - - return 1 + min_length - - def get_outgoing_edges(node_id: str) -> List[Edge]: - """Get outgoing edges sorted by the shortest possible path length""" - out_edges = [edge for edge in edges if edge.source == node_id] - # Sort edges by the length of the shortest possible path from their target - return sorted(out_edges, key=lambda e: calculate_path_length(e.target)) - - def create_step( - node_id: str, - branch_id: str, - depth: int, - input_data: Dict[str, Any], - is_input_node: bool, - outputs: Dict[str, Any], - node_params: Optional[Dict[str, Any]] = None, - ) -> FlowStep: - return FlowStep( - nodeId=node_id, - params=node_params, - inputs={} if is_input_node else input_data, - outputs=outputs, - type="type" if is_input_node else "scanner", - status="pending", - branchId=branch_id, - depth=depth, - ) - - def explore_branch( - current_node_id: str, - branch_id: str, - branch_name: str, - depth: int, - input_data: Dict[str, Any], - path: List[str], - branch_visited: set, - steps: List[FlowStep], - parent_outputs: Dict[str, Any] = None, - ) -> None: - nonlocal branch_counter - - # Skip if node is already in current path (cycle detection) - if current_node_id in path: - return - - current_node = node_map.get(current_node_id) - if not current_node: - return - - # Process node outputs - is_input_node = current_node.data.get("type") == "type" - if is_input_node: - outputs_array = current_node.data["outputs"].get("properties", []) - first_output_name = ( - outputs_array[0].get("name", "output") if outputs_array else "output" - ) - current_outputs = {first_output_name: initial_value} - else: - # Check if we already have outputs for this scanner - if current_node_id in scanner_outputs: - current_outputs = scanner_outputs[current_node_id] - else: - current_outputs = process_node_data(current_node, input_data) - # Store the outputs for future use - scanner_outputs[current_node_id] = current_outputs - - # Extract node parameters - node_params = current_node.data.get("params", {}) - - # Create and add current step - current_step = create_step( - current_node_id, - branch_id, - depth, - input_data, - is_input_node, - current_outputs, - node_params, - ) - steps.append(current_step) - path.append(current_node_id) - branch_visited.add(current_node_id) - - # Get all outgoing edges sorted by path length - out_edges = get_outgoing_edges(current_node_id) - - if not out_edges: - # Leaf node reached, save the branch - branches.append(FlowBranch(id=branch_id, name=branch_name, steps=steps[:])) - else: - # Process each outgoing edge in order of shortest path - for i, edge in enumerate(out_edges): - if edge.target in path: # Skip if would create cycle - continue - - # Prepare next node's input - output_key = edge.sourceHandle - if not output_key and current_outputs: - output_key = list(current_outputs.keys())[0] - - output_value = current_outputs.get(output_key) if output_key else None - if output_value is None and parent_outputs: - output_value = ( - parent_outputs.get(output_key) if output_key else None - ) - - next_input = {edge.targetHandle or "input": output_value} - - if i == 0: - # Continue in same branch (will be shortest path) - explore_branch( - edge.target, - branch_id, - branch_name, - depth + 1, - next_input, - path, - branch_visited, - steps, - current_outputs, - ) - else: - # Create new branch starting from current node - branch_counter += 1 - new_branch_id = f"{branch_id}-{branch_counter}" - new_branch_name = f"{branch_name} (Branch {branch_counter})" - new_steps = steps[: len(steps)] # Copy steps up to current node - new_branch_visited = ( - branch_visited.copy() - ) # Create new visited set for the branch - explore_branch( - edge.target, - new_branch_id, - new_branch_name, - depth + 1, - next_input, - path[:], # Create new path copy for branch - new_branch_visited, - new_steps, - current_outputs, - ) - - # Backtrack: remove current node from path and remove its step - path.pop() - steps.pop() - - # Start exploration from each input node - for index, input_node in enumerate(input_nodes): - branch_id = f"branch-{index}" - branch_name = f"Flow {index + 1}" if len(input_nodes) > 1 else "Main Flow" - explore_branch( - input_node.id, - branch_id, - branch_name, - 0, - {}, - [], # Use list for path to maintain order - set(), # Use set for visited to check membership - [], - None, - ) - - # Sort branches by length (number of steps) - branches.sort(key=lambda branch: len(branch.steps)) - return branches - - -def process_node_data(node: Node, inputs: Dict[str, Any]) -> Dict[str, Any]: - """Traite les données de nœud en fonction du type de nœud et des entrées""" - outputs = {} - output_types = node.data["outputs"].get("properties", []) - - for output in output_types: - output_name = output.get("name", "output") - class_name = node.data.get("class_name", "") - # For simulation purposes, we'll return a placeholder value based on the scanner type - if class_name in ["ReverseResolveScanner", "ResolveScanner"]: - # IP/Domain resolution scanners - outputs[output_name] = ( - "192.168.1.1" if "ip" in output_name.lower() else "example.com" - ) - elif class_name == "SubdomainScanner": - # Subdomain scanner - outputs[output_name] = f"sub.{inputs.get('input', 'example.com')}" - - elif class_name == "WhoisScanner": - # WHOIS scanner - outputs[output_name] = { - "domain": inputs.get("input", "example.com"), - "registrar": "Example Registrar", - "creation_date": "2020-01-01", - } - - elif class_name == "GeolocationScanner": - # Geolocation scanner - outputs[output_name] = { - "country": "France", - "city": "Paris", - "coordinates": {"lat": 48.8566, "lon": 2.3522}, - } - - elif class_name == "MaigretScanner": - # Social media scanner - outputs[output_name] = { - "username": inputs.get("input", "user123"), - "platforms": ["twitter", "github", "linkedin"], - } - - elif class_name == "HoleheScanner": - # Email verification scanner - outputs[output_name] = { - "email": inputs.get("input", "user@example.com"), - "exists": True, - "platforms": ["gmail", "github"], - } - - elif class_name == "SireneScanner": - # Organization scanner - outputs[output_name] = { - "name": inputs.get("input", "Example Corp"), - "siret": "12345678901234", - "address": "1 Example Street", - } - - else: - # For unknown scanners, pass through the input - outputs[output_name] = inputs.get("input") or f"transformed_{output_name}" - - return outputs diff --git a/flowsint-api/app/api/schemas/flow.py b/flowsint-api/app/api/schemas/flow.py new file mode 100644 index 0000000..5912698 --- /dev/null +++ b/flowsint-api/app/api/schemas/flow.py @@ -0,0 +1,29 @@ +from .base import ORMBase +from pydantic import UUID4, BaseModel +from typing import Optional +from datetime import datetime +from typing import List, Optional, Dict, Any + + +class FlowCreate(BaseModel): + name: str + description: Optional[str] = None + category: Optional[List[str]] = None + flow_schema: Optional[Dict[str, Any]] = None + + +class FlowRead(ORMBase): + id: UUID4 + name: str + description: Optional[str] + category: Optional[List[str]] + flow_schema: Optional[Dict[str, Any]] + created_at: datetime + last_updated_at: datetime + + +class FlowUpdate(BaseModel): + name: Optional[str] = None + description: Optional[str] = None + category: Optional[List[str]] = None + flow_schema: Optional[Dict[str, Any]] = None diff --git a/flowsint-api/app/api/schemas/investigation.py b/flowsint-api/app/api/schemas/investigation.py index cccb1c0..d9b5d89 100644 --- a/flowsint-api/app/api/schemas/investigation.py +++ b/flowsint-api/app/api/schemas/investigation.py @@ -3,6 +3,8 @@ from pydantic import UUID4, BaseModel from typing import Optional from datetime import datetime from .sketch import SketchRead +from .analysis import AnalysisRead +from .profile import ProfileRead class InvestigationCreate(BaseModel): @@ -20,7 +22,9 @@ class InvestigationRead(ORMBase): owner_id: Optional[UUID4] last_updated_at: datetime status: str + owner: Optional[ProfileRead] = None sketches: list[SketchRead] = [] + analyses: list[AnalysisRead] = [] class InvestigationProfileCreate(BaseModel): diff --git a/flowsint-api/app/api/schemas/transform.py b/flowsint-api/app/api/schemas/transform.py index bc28250..455266e 100644 --- a/flowsint-api/app/api/schemas/transform.py +++ b/flowsint-api/app/api/schemas/transform.py @@ -1,7 +1,6 @@ from .base import ORMBase from pydantic import UUID4, BaseModel from typing import Optional -from datetime import datetime from typing import List, Optional, Dict, Any @@ -9,21 +8,19 @@ class TransformCreate(BaseModel): name: str description: Optional[str] = None category: Optional[List[str]] = None - transform_schema: Optional[Dict[str, Any]] = None class TransformRead(ORMBase): id: UUID4 name: str + class_name: str description: Optional[str] category: Optional[List[str]] - transform_schema: Optional[Dict[str, Any]] - created_at: datetime - last_updated_at: datetime + flow_schema: Optional[Dict[str, Any]] class TransformUpdate(BaseModel): name: Optional[str] = None + class_name: str = None description: Optional[str] = None category: Optional[List[str]] = None - transform_schema: Optional[Dict[str, Any]] = None diff --git a/flowsint-api/app/main.py b/flowsint-api/app/main.py index 8a91b89..e33c641 100644 --- a/flowsint-api/app/main.py +++ b/flowsint-api/app/main.py @@ -1,7 +1,6 @@ from fastapi import FastAPI from flowsint_core.core.graph_db import Neo4jConnection import os -from typing import List from dotenv import load_dotenv from fastapi.middleware.cors import CORSMiddleware @@ -10,6 +9,7 @@ from app.api.routes import auth from app.api.routes import investigations from app.api.routes import sketches from app.api.routes import transforms +from app.api.routes import flows from app.api.routes import events from app.api.routes import analysis from app.api.routes import chat @@ -58,6 +58,7 @@ app.include_router( investigations.router, prefix="/api/investigations", tags=["investigations"] ) app.include_router(transforms.router, prefix="/api/transforms", tags=["transforms"]) +app.include_router(flows.router, prefix="/api/flows", tags=["flows"]) app.include_router(events.router, prefix="/api/events", tags=["events"]) app.include_router(analysis.router, prefix="/api/analyses", tags=["analyses"]) app.include_router(chat.router, prefix="/api/chats", tags=["chats"]) diff --git a/flowsint-api/app/models/models.py b/flowsint-api/app/models/models.py index 86cd56f..34094d6 100644 --- a/flowsint-api/app/models/models.py +++ b/flowsint-api/app/models/models.py @@ -50,6 +50,7 @@ class Investigation(Base): sketches = relationship("Sketch", back_populates="investigation") analyses = relationship("Analysis", back_populates="investigation") chats = relationship("Chat", back_populates="investigation") + owner = relationship("Profile", foreign_keys=[owner_id]) __table_args__ = ( Index("idx_investigations_id", "id"), Index("idx_investigations_owner_id", "owner_id"), @@ -194,8 +195,8 @@ class SketchesProfiles(Base): ) -class Transform(Base): - __tablename__ = "transforms" +class Flow(Base): + __tablename__ = "flows" id: Mapped[uuid.UUID] = mapped_column( PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4 @@ -203,7 +204,7 @@ class Transform(Base): name = mapped_column(Text, nullable=False) description = mapped_column(Text, nullable=True) category = mapped_column(ARRAY(Text), nullable=True) - transform_schema = mapped_column(JSON, nullable=True) + flow_schema = mapped_column(JSON, nullable=True) created_at = mapped_column(DateTime(timezone=True), server_default=func.now()) last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now()) diff --git a/flowsint-api/app/utils.py b/flowsint-api/app/utils.py index 238eed7..5a6441a 100644 --- a/flowsint-api/app/utils.py +++ b/flowsint-api/app/utils.py @@ -159,7 +159,7 @@ def resolve_type(details: dict, schema_context: dict = None) -> str: return "any" -def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]: +def extract_input_schema_flow(model: Type[BaseModel]) -> Dict[str, Any]: adapter = TypeAdapter(model) schema = adapter.json_schema() @@ -225,7 +225,7 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]: if scanner_node and scanner_node["data"]["type"] == "scanner": scanners.append( { - "scanner_name": scanner_node["data"]["name"], + "transform_name": scanner_node["data"]["name"], "module": scanner_node["data"]["module"], "input": source_handle, "output": target_handle, @@ -238,7 +238,7 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]: "outputs": input_output, }, "scanners": scanners, - "scanner_names": [scanner["scanner_name"] for scanner in scanners], + "transform_names": [scanner["transform_name"] for scanner in scanners], } diff --git a/flowsint-app/src/renderer/index.html b/flowsint-app/src/renderer/index.html index c41e657..153b424 100644 --- a/flowsint-app/src/renderer/index.html +++ b/flowsint-app/src/renderer/index.html @@ -1,11 +1,11 @@ - + diff --git a/flowsint-app/src/renderer/src/api/flow-service.ts b/flowsint-app/src/renderer/src/api/flow-service.ts new file mode 100644 index 0000000..0efb6f3 --- /dev/null +++ b/flowsint-app/src/renderer/src/api/flow-service.ts @@ -0,0 +1,54 @@ +import { fetchWithAuth } from './api'; + +export const flowService = { + get: async (type?: string): Promise => { + const url = type ? `/api/flows?category=${type}` : "/api/flows" + return fetchWithAuth(url, { + method: 'GET', + }); + }, + getById: async (flowId: string): Promise => { + return fetchWithAuth(`/api/flows/${flowId}`, { + method: 'GET', + }); + }, + create: async (body: BodyInit): Promise => { + return fetchWithAuth(`/api/flows/create`, { + method: 'POST', + body: body + }); + }, + update: async (flowId: string, body: BodyInit): Promise => { + return fetchWithAuth(`/api/flows/${flowId}`, { + method: 'PUT', + body: body + }); + }, + compute: async (flowId: string, body: BodyInit): Promise => { + return fetchWithAuth(`/api/flows/${flowId}/compute`, { + method: 'POST', + body: body + }); + }, + delete: async (flowId: string): Promise => { + return fetchWithAuth(`/api/flows/${flowId}`, { + method: 'DELETE', + }); + }, + launch: async (flowId: string, body: BodyInit): Promise => { + return fetchWithAuth(`/api/flows/${flowId}/launch`, { + method: 'POST', + body: body + }); + }, + getRawMaterial: async (): Promise => { + return fetchWithAuth(`/api/flows/raw_materials`, { + method: 'GET', + }); + }, + getRawMaterialForType: async (type: string): Promise => { + return fetchWithAuth(`/api/flows/input_type/${type}`, { + method: 'GET', + }); + }, +}; \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/api/transfrom-service.ts b/flowsint-app/src/renderer/src/api/transfrom-service.ts index 9322ca0..88c3bf0 100644 --- a/flowsint-app/src/renderer/src/api/transfrom-service.ts +++ b/flowsint-app/src/renderer/src/api/transfrom-service.ts @@ -1,6 +1,5 @@ import { fetchWithAuth } from './api'; - export const transformService = { get: async (type?: string): Promise => { const url = type ? `/api/transforms?category=${type}` : "/api/transforms" @@ -8,48 +7,10 @@ export const transformService = { method: 'GET', }); }, - getById: async (transformId: string): Promise => { - return fetchWithAuth(`/api/transforms/${transformId}`, { - method: 'GET', - }); - }, - create: async (body: BodyInit): Promise => { - return fetchWithAuth(`/api/transforms/create`, { + launch: async (transformName: string, body: BodyInit): Promise => { + return fetchWithAuth(`/api/transforms/${transformName}/launch`, { method: 'POST', body: body }); }, - update: async (transformId: string, body: BodyInit): Promise => { - return fetchWithAuth(`/api/transforms/${transformId}`, { - method: 'PUT', - body: body - }); - }, - compute: async (transformId: string, body: BodyInit): Promise => { - return fetchWithAuth(`/api/transforms/${transformId}/compute`, { - method: 'POST', - body: body - }); - }, - delete: async (transformId: string): Promise => { - return fetchWithAuth(`/api/transforms/${transformId}`, { - method: 'DELETE', - }); - }, - launch: async (transformId: string, body: BodyInit): Promise => { - return fetchWithAuth(`/api/transforms/${transformId}/launch`, { - method: 'POST', - body: body - }); - }, - getRawMaterial: async (): Promise => { - return fetchWithAuth(`/api/transforms/raw_materials`, { - method: 'GET', - }); - }, - getRawMaterialForType: async (type: string): Promise => { - return fetchWithAuth(`/api/transforms/input_type/${type}`, { - method: 'GET', - }); - }, }; \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/command.tsx b/flowsint-app/src/renderer/src/components/command.tsx index 6e938c1..5fe157d 100644 --- a/flowsint-app/src/renderer/src/components/command.tsx +++ b/flowsint-app/src/renderer/src/components/command.tsx @@ -209,7 +209,7 @@ export function Command() { - setOpen(false)} to="/dashboard/transforms"> + setOpen(false)} to="/dashboard/flows"> Transforms diff --git a/flowsint-app/src/renderer/src/components/transforms/context-menu.tsx b/flowsint-app/src/renderer/src/components/flows/context-menu.tsx similarity index 84% rename from flowsint-app/src/renderer/src/components/transforms/context-menu.tsx rename to flowsint-app/src/renderer/src/components/flows/context-menu.tsx index 95a288b..ffb1775 100644 --- a/flowsint-app/src/renderer/src/components/transforms/context-menu.tsx +++ b/flowsint-app/src/renderer/src/components/flows/context-menu.tsx @@ -2,10 +2,10 @@ import { useCallback } from 'react'; import { Pencil, Trash } from 'lucide-react'; import { GraphNode } from '@/stores/graph-store'; import BaseContextMenu from '@/components/xyflow/context-menu'; -import { TransformNode, useTransformStore } from '@/stores/transform-store'; +import { FlowNode, useFlowStore } from '@/stores/flow-store'; interface GraphContextMenuProps { - node: GraphNode | TransformNode; + node: GraphNode | FlowNode; top?: number; left?: number; right?: number; @@ -34,15 +34,15 @@ export default function ContextMenu({ ...props }: GraphContextMenuProps) { - const setOpenParamsDialog = useTransformStore(s => s.setOpenParamsDialog) - const deleteNode = useTransformStore(s => s.deleteNode) + const setOpenParamsDialog = useFlowStore(s => s.setOpenParamsDialog) + const deleteNode = useFlowStore(s => s.deleteNode) const handleOpenParamsModal = useCallback(() => { - setOpenParamsDialog(true, node as TransformNode) + setOpenParamsDialog(true, node as FlowNode) setMenu(null) }, [setOpenParamsDialog, node, setMenu]) - const handleDeleteTransform = useCallback(() => { + const handleDeleteFlow = useCallback(() => { deleteNode(node.id as string) setMenu(null) }, [deleteNode, node.id, setMenu]) @@ -73,7 +73,7 @@ export default function ContextMenu({ Edit -

Save Transform

+

Save flow

 @@ -50,14 +50,14 @@ export function FlowControls({ variant="outline" size="icon" className="bg-card" - onClick={handleDeleteTransform} + onClick={handleDeleteFlow} disabled={loading} > -

Delete Transform

+

Delete flow

 )} diff --git a/flowsint-app/src/renderer/src/components/transforms/editor.tsx b/flowsint-app/src/renderer/src/components/flows/editor.tsx similarity index 80% rename from flowsint-app/src/renderer/src/components/transforms/editor.tsx rename to flowsint-app/src/renderer/src/components/flows/editor.tsx index 505bef3..973457b 100644 --- a/flowsint-app/src/renderer/src/components/transforms/editor.tsx +++ b/flowsint-app/src/renderer/src/components/flows/editor.tsx @@ -25,11 +25,11 @@ import { categoryColors } from "./scanner-data" import { FlowControls } from "./controls" import { getDagreLayoutedElements } from "@/lib/utils" import { toast } from "sonner" -import { TransformModal } from "./save-modal" +import { SaveModal } from "./save-modal" import { useConfirm } from "@/components/use-confirm-dialog" import { useParams, useRouter } from "@tanstack/react-router" -import { transformService } from "@/api/transfrom-service" -import { useTransformStore, type TransformNode, type TransformEdge } from "@/stores/transform-store" +import { flowService } from "@/api/flow-service" +import { useFlowStore, type FlowNode, type FlowEdge } from "@/stores/flow-store" import type { CSSProperties } from 'react' import { Select, @@ -40,7 +40,7 @@ import { } from "@/components/ui/select" import { useTheme } from "../theme-provider" import ParamsDialog from "./params-dialog" -import TransformSheet from "./transform-sheet" +import FlowSheet from "./flow-sheet" import ContextMenu from "./context-menu" const nodeTypes: NodeTypes = { @@ -48,11 +48,11 @@ const nodeTypes: NodeTypes = { type: TypeNode, } -interface TransformEditorProps { +interface FlowEditorProps { theme?: ColorMode - initialEdges: TransformEdge[] - initialNodes: TransformNode[] - transform?: any + initialEdges: FlowEdge[] + initialNodes: FlowNode[] + flow?: any } const defaultEdgeStyle: CSSProperties = { stroke: "#64748b" } @@ -63,14 +63,14 @@ const defaultMarkerEnd: EdgeMarker = { color: "#64748b", } -const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform }: TransformEditorProps) => { +const FlowEditor = memo(({ initialEdges, initialNodes, theme, flow }: FlowEditorProps) => { // #### React Flow and UI State #### const { fitView, zoomIn, zoomOut, setCenter } = useReactFlow() const reactFlowWrapper = useRef(null) const [reactFlowInstance, setReactFlowInstance] = useState(null) const router = useRouter() const { confirm } = useConfirm() - const { transformId } = useParams({ strict: false }) + const { flowId } = useParams({ strict: false }) const [showModal, setShowModal] = useState(false) const hasInitialized = useRef(false) @@ -78,22 +78,22 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform const [isSimulating, setIsSimulating] = useState(false) const [currentStepIndex, setCurrentStepIndex] = useState(0) const [simulationSpeed, setSimulationSpeed] = useState(1000) // ms per step - const [transformBranches, setTransformsBranches] = useState([]) + const [flowBranches, setFlowBranches] = useState([]) // #### Transform Store State #### - const nodes = useTransformStore(state => state.nodes) - const edges = useTransformStore(state => state.edges) - const loading = useTransformStore(state => state.loading) - const setNodes = useTransformStore(state => state.setNodes) - const setEdges = useTransformStore(state => state.setEdges) - const onNodesChange = useTransformStore(state => state.onNodesChange) - const onEdgesChange = useTransformStore(state => state.onEdgesChange) - const onConnect = useTransformStore(state => state.onConnect) - const setSelectedNode = useTransformStore(state => state.setSelectedNode) - const setLoading = useTransformStore(state => state.setLoading) + const nodes = useFlowStore(state => state.nodes) + const edges = useFlowStore(state => state.edges) + const loading = useFlowStore(state => state.loading) + const setNodes = useFlowStore(state => state.setNodes) + const setEdges = useFlowStore(state => state.setEdges) + const onNodesChange = useFlowStore(state => state.onNodesChange) + const onEdgesChange = useFlowStore(state => state.onEdgesChange) + const onConnect = useFlowStore(state => state.onConnect) + const setSelectedNode = useFlowStore(state => state.setSelectedNode) + const setLoading = useFlowStore(state => state.setLoading) const [menu, setMenu] = useState<{ - node: TransformNode; + node: FlowNode; top?: number; left?: number; right?: number; @@ -129,7 +129,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform const relativeY = event.clientY - pane.top; setMenu({ - node: node as TransformNode, + node: node as FlowNode, rawTop: relativeY, rawLeft: relativeX, wrapperWidth: pane.width, @@ -179,7 +179,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform x: event.clientX - reactFlowBounds.left, y: event.clientY - reactFlowBounds.top, }) - const newNode: TransformNode = { + const newNode: FlowNode = { id: `${scannerData.name}-${Date.now()}`, type: scannerData.type === "type" ? "type" : "scanner", position, @@ -217,7 +217,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform // #### Node Interaction Handlers #### const onNodeClick: NodeMouseHandler = useCallback( (_: React.MouseEvent, node: Node) => { - const typedNode = node as TransformNode + const typedNode = node as FlowNode setSelectedNode(typedNode) // const nodeWidth = typedNode.measured?.width ?? 0 // const nodeHeight = typedNode.measured?.height ?? 0 @@ -250,11 +250,11 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform key: node.data.class_name || node.id, }, })), - edges as TransformEdge[], + edges as FlowEdge[], { direction: "LR" } ) - setNodes(layouted.nodes as TransformNode[]) - setEdges(layouted.edges as TransformEdge[]) + setNodes(layouted.nodes as FlowNode[]) + setEdges(layouted.edges as FlowEdge[]) window.requestAnimationFrame(() => { fitView() }) @@ -262,7 +262,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform }, [nodes, edges, setNodes, setEdges, fitView]) // #### Transform CRUD Operations #### - const saveTransform = useCallback( + const saveFlow = useCallback( async (name: string, description: string) => { setLoading(true) try { @@ -276,22 +276,22 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform name: name, description: description, category: [inputType], - transform_schema: { + flow_schema: { nodes, edges, }, }) - if (transformId) { - newTransform = await transformService.update(transformId, body) + if (flowId) { + newTransform = await flowService.update(flowId, body) } else { - newTransform = await transformService.create(body) + newTransform = await flowService.create(body) if (!newTransform) { toast.error("Error creating transform") return } } toast.success("Transform saved successfully.") - newTransform && !transformId && router.navigate({ to: `/dashboard/transforms/${newTransform.id}` }) + newTransform && !flowId && router.navigate({ to: `/dashboard/flows/${newTransform.id}` }) } catch (error) { toast.error("Error saving transform" + JSON.stringify(error)) } finally { @@ -299,37 +299,37 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform setShowModal(false) } }, - [nodes, edges, transformId, router, setLoading], + [nodes, edges, flowId, router, setLoading], ) - const handleSaveTransform = useCallback(async () => { - if (!transformId) { + const handleSaveFlow = useCallback(async () => { + if (!flowId) { setShowModal(true) } else { - await saveTransform(transform?.name || "", transform?.description || "") + await saveFlow(flow?.name || "", flow?.description || "") } - }, [transformId, saveTransform, transform]) + }, [flowId, saveFlow, flow]) - const handleDeleteTransform = useCallback(async () => { - if (!transformId) return + const handleDeleteFlow = useCallback(async () => { + if (!flowId) return if ( await confirm({ title: "Are you sure you want to delete this flow ?", - message: "All of the transforms settings will be lost.", + message: "All of the flow's settings will be lost.", }) ) { setLoading(true) - await transformService.delete(transformId) - router.navigate({ to: "/dashboard/transforms" }) - toast.success("Transform deleted successfully.") + await flowService.delete(flowId) + router.navigate({ to: "/dashboard/flows" }) + toast.success("Flow deleted successfully.") setLoading(false) } - }, [transformId, confirm, setLoading]) + }, [flowId, confirm, setLoading]) // #### Flow Computation #### const handleComputeFlow = useCallback(async () => { - if (!transformId) { - toast.error("Save the transform first to compute the flow.") + if (!flowId) { + toast.error("Save the flow first to compute it.") return } @@ -340,19 +340,19 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform edges, initialValue: "domain" } - const response = await transformService.compute(transformId, JSON.stringify(body)) - setTransformsBranches(response.transformBranches) + const response = await flowService.compute(flowId, JSON.stringify(body)) + setFlowBranches(response.flowBranches) startSimulation() } catch (error) { toast.error("Error computing flow") } finally { setLoading(false) } - }, [transformId, nodes, edges]) + }, [flowId, nodes, edges]) // #### Simulation State Management #### // Update the updateNodeState function with proper types - const updateNodeState = useCallback((nds: TransformNode[], nodeId: string, state: 'pending' | 'processing' | 'completed' | 'error') => { + const updateNodeState = useCallback((nds: FlowNode[], nodeId: string, state: 'pending' | 'processing' | 'completed' | 'error') => { return nds.map((node) => { // focus on node if (node.id === nodeId) { @@ -381,7 +381,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform let timer: NodeJS.Timeout - const totalSteps = transformBranches.reduce((sum, branch) => sum + branch.steps.length, 0) + const totalSteps = flowBranches.reduce((sum, branch) => sum + branch.steps.length, 0) if (currentStepIndex < totalSteps) { // Find the current branch and step @@ -390,8 +390,8 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform let stepIndex = 0 let currentStepCount = 0 - for (let i = 0; i < transformBranches.length; i++) { - const branch = transformBranches[i] + for (let i = 0; i < flowBranches.length; i++) { + const branch = flowBranches[i] if (currentStepCount + branch.steps.length > currentStepIndex) { branchIndex = i stepIndex = currentStepIndex - currentStepCount @@ -402,13 +402,13 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform } if (stepFound) { - const currentStep = transformBranches[branchIndex].steps[stepIndex] + const currentStep = flowBranches[branchIndex].steps[stepIndex] // Update node states with proper types - setNodes((nds: TransformNode[]) => updateNodeState(nds, currentStep.nodeId, "processing")) + setNodes((nds: FlowNode[]) => updateNodeState(nds, currentStep.nodeId, "processing")) // Update edges with proper types - setEdges((eds: TransformEdge[]) => { + setEdges((eds: FlowEdge[]) => { return eds.map((edge) => ({ ...edge, style: { @@ -422,7 +422,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform // After delay, mark as completed and move to next step timer = setTimeout(() => { - setNodes((nds: TransformNode[]) => updateNodeState(nds, currentStep.nodeId, "completed")) + setNodes((nds: FlowNode[]) => updateNodeState(nds, currentStep.nodeId, "completed")) setCurrentStepIndex((prev) => prev + 1) }, simulationSpeed) } @@ -433,12 +433,12 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform } return () => clearTimeout(timer) - }, [isSimulating, currentStepIndex, simulationSpeed, loading, transformBranches, updateNodeState, setCurrentStepIndex]) + }, [isSimulating, currentStepIndex, simulationSpeed, loading, flowBranches, updateNodeState, setCurrentStepIndex]) // #### Simulation Control Functions #### const startSimulation = () => { // Reset all nodes to pending state - setNodes((nds: TransformNode[]) => + setNodes((nds: FlowNode[]) => nds.map((node) => ({ ...node, data: { @@ -449,7 +449,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform ) // Reset all edges - setEdges((eds: TransformEdge[]) => + setEdges((eds: FlowEdge[]) => eds.map((edge) => ({ ...edge, style: { @@ -473,7 +473,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform setIsSimulating(false) // Mark all nodes as completed - setNodes((nds: TransformNode[]) => + setNodes((nds: FlowNode[]) => nds.map((node) => ({ ...node, data: { @@ -484,7 +484,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform ) // Reset edge styling - setEdges((eds: TransformEdge[]) => + setEdges((eds: FlowEdge[]) => eds.map((edge) => ({ ...edge, style: { @@ -496,7 +496,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform })) ) - const totalSteps = transformBranches.reduce((sum, branch) => sum + branch.steps.length, 0) + const totalSteps = flowBranches.reduce((sum, branch) => sum + branch.steps.length, 0) setCurrentStepIndex(totalSteps) } @@ -505,7 +505,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform setCurrentStepIndex(0) // Reset all nodes - setNodes((nds: TransformNode[]) => + setNodes((nds: FlowNode[]) => nds.map((node) => ({ ...node, data: { @@ -516,7 +516,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform ) // Reset all edges - setEdges((eds: TransformEdge[]) => + setEdges((eds: FlowEdge[]) => eds.map((edge) => ({ ...edge, style: { @@ -550,7 +550,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform proOptions={{ hideAttribution: true }} colorMode={theme} > - {transformId && + {flowId && {isSimulating ? ( - + setSearchQuery(e.target.value)} /> - {filteredTransforms.length > 0 ? ( + {filteredflows.length > 0 ? (
    - {filteredTransforms.map((transform: any) => ( -
  • + {filteredflows.map((flow: any) => ( +
  • -
    {transform.name || "(Unnamed transform)"}
    +
    {flow.name || "(Unnamed flow)"}
    - {transform.category && ( + {flow.category && ( - {Array.isArray(transform.category) ? transform.category.join(", ") : transform.category} + {Array.isArray(flow.category) ? flow.category.join(", ") : flow.category} )} - {transform.last_updated_at && ( + {flow.last_updated_at && ( <> - {Boolean(transform.category.length) && •} + {Boolean(flow.category.length) && •}
    - {formatDistanceToNow(new Date(transform.last_updated_at), { addSuffix: true })} + {formatDistanceToNow(new Date(flow.last_updated_at), { addSuffix: true })}
    )} @@ -84,11 +84,11 @@ const TransformsList = () => {
) : (
- {searchQuery ? "No matching transforms found" : "No transforms found"} + {searchQuery ? "No matching flows found" : "No flows found"}
)} ) } -export default TransformsList +export default FlowsList diff --git a/flowsint-app/src/renderer/src/components/transforms/transform-name-panel.tsx b/flowsint-app/src/renderer/src/components/flows/flow-name-panel.tsx similarity index 77% rename from flowsint-app/src/renderer/src/components/transforms/transform-name-panel.tsx rename to flowsint-app/src/renderer/src/components/flows/flow-name-panel.tsx index 4290ea7..78197aa 100644 --- a/flowsint-app/src/renderer/src/components/transforms/transform-name-panel.tsx +++ b/flowsint-app/src/renderer/src/components/flows/flow-name-panel.tsx @@ -3,10 +3,10 @@ import { useState, useEffect, type KeyboardEvent, useRef } from "react" import { Panel } from "@xyflow/react" import { toast } from "sonner" import { Card } from "@/components/ui/card" -import { transformService } from "@/api/transfrom-service" +import { flowService } from "@/api/flow-service" -interface TransformDetailsPanelProps { - transform?: { +interface FlowDetailsPanelProps { + flow?: { id: string name: string description?: string @@ -15,9 +15,9 @@ interface TransformDetailsPanelProps { disabled?: boolean } -export function TransformDetailsPanel({ transform, onUpdate, disabled = false }: TransformDetailsPanelProps) { - const [name, setName] = useState(transform?.name || "My Transform") - const [description, setDescription] = useState(transform?.description || "") +export function FlowDetailsPanel({ flow, onUpdate, disabled = false }: FlowDetailsPanelProps) { + const [name, setName] = useState(flow?.name || "My Flow") + const [description, setDescription] = useState(flow?.description || "") const [isEditingName, setIsEditingName] = useState(false) const [isEditingDesc, setIsEditingDesc] = useState(false) const [isSaving, setIsSaving] = useState(false) @@ -25,13 +25,13 @@ export function TransformDetailsPanel({ transform, onUpdate, disabled = false }: const descInputRef = useRef(null) useEffect(() => { - if (transform?.name) { - setName(transform.name) + if (flow?.name) { + setName(flow.name) } - if (transform?.description !== undefined) { - setDescription(transform.description) + if (flow?.description !== undefined) { + setDescription(flow.description) } - }, [transform?.name, transform?.description]) + }, [flow?.name, flow?.description]) useEffect(() => { if (isEditingName && nameInputRef.current) { @@ -46,14 +46,14 @@ export function TransformDetailsPanel({ transform, onUpdate, disabled = false }: }, [isEditingDesc]) const handleSaveField = async (field: "name" | "description", value: string) => { - if (!transform?.id) return + if (!flow?.id) return const trimmedValue = value.trim() if (field === "name" && trimmedValue === "") return if ( - (field === "name" && trimmedValue === transform.name) || - (field === "description" && trimmedValue === transform.description) + (field === "name" && trimmedValue === flow.name) || + (field === "description" && trimmedValue === flow.description) ) { field === "name" ? setIsEditingName(false) : setIsEditingDesc(false) return @@ -61,18 +61,17 @@ export function TransformDetailsPanel({ transform, onUpdate, disabled = false }: setIsSaving(true) try { const updates = { [field]: trimmedValue } - console.log(updates) - await transformService.update(transform.id, JSON.stringify(updates)) + await flowService.update(flow.id, JSON.stringify(updates)) if (onUpdate) { onUpdate(updates) } toast.success(`${field === "name" ? "Name" : "Description"} updated.`) } catch (error) { - toast.error(`Failed to update transform ${field}`) + toast.error(`Failed to update flow ${field}`) if (field === "name") { - setName(transform.name) + setName(flow.name) } else { - setDescription(transform.description || "") + setDescription(flow.description || "") } } finally { setIsSaving(false) @@ -87,10 +86,10 @@ export function TransformDetailsPanel({ transform, onUpdate, disabled = false }: } else if (e.key === "Escape") { if (field === "name") { setIsEditingName(false) - setName(transform?.name || "My Transform") + setName(flow?.name || "My flow") } else { setIsEditingDesc(false) - setDescription(transform?.description || "") + setDescription(flow?.description || "") } } } @@ -108,14 +107,14 @@ export function TransformDetailsPanel({ transform, onUpdate, disabled = false }: onKeyDown={(e) => handleKeyDown(e, "name", name)} disabled={disabled || isSaving} className="!text-xl font-semibold bg-transparent border-b border-gray-300 focus:border-primary focus:outline-none !px-1 !py-0.5 w-full" - placeholder="Enter transform name" + placeholder="Enter flow name" /> ) : (

!disabled && setIsEditingName(true)} > - {name || "My Transform"} + {name || "My flow"}

)} diff --git a/flowsint-app/src/renderer/src/components/transforms/transform-navigation.tsx b/flowsint-app/src/renderer/src/components/flows/flow-navigation.tsx similarity index 81% rename from flowsint-app/src/renderer/src/components/transforms/transform-navigation.tsx rename to flowsint-app/src/renderer/src/components/flows/flow-navigation.tsx index 754fc06..0ea35bb 100644 --- a/flowsint-app/src/renderer/src/components/transforms/transform-navigation.tsx +++ b/flowsint-app/src/renderer/src/components/flows/flow-navigation.tsx @@ -1,19 +1,19 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" import { UserPlus, Users } from "lucide-react" import RawMaterial from "./raw-material" -import TransformsList from "./transforms-list" +import TransformsList from "./flow-list" import { useLayoutStore } from "@/stores/layout-store" import { useParams } from "@tanstack/react-router" import { useEffect } from "react" const TransformNavigation = () => { - const { transformId } = useParams({ strict: false }) + const { flowId } = useParams({ strict: false }) const activeTransformTab = useLayoutStore((s) => s.activeTransformTab) const setActiveTransformTab = useLayoutStore((s) => s.setActiveTransformTab) useEffect(() => { - setActiveTransformTab(transformId ? "items" : "transforms") - }, [setActiveTransformTab, transformId]) + setActiveTransformTab(flowId ? "items" : "transforms") + }, [setActiveTransformTab, flowId]) return (
@@ -24,7 +24,7 @@ const TransformNavigation = () => { className="w-full h-full flex flex-col gap-0 min-h-0"> Transforms - {transformId && Items} + {flowId && Items} { > - {transformId && + {flowId && void }) => { - const openTransformSheet = useTransformStore(state => state.openTransformSheet) - const setOpenTransformSheet = useTransformStore(state => state.setOpenTransformSheet) - const selectedNode = useTransformStore(state => state.selectedNode) - const setNodes = useTransformStore(state => state.setNodes) - const setEdges = useTransformStore(state => state.setEdges) +const FlowSheet = ({ onLayout }: { onLayout: () => void }) => { + const openFlowSheet = useFlowStore(state => state.openFlowSheet) + const setOpenFlowSheet = useFlowStore(state => state.setOpenFlowSheet) + const selectedNode = useFlowStore(state => state.selectedNode) + const setNodes = useFlowStore(state => state.setNodes) + const setEdges = useFlowStore(state => state.setEdges) const { data: materials, isLoading, error } = useQuery({ queryKey: ["raw_material", selectedNode?.data.outputs.type], enabled: !!selectedNode?.data.outputs.type, - queryFn: () => transformService.getRawMaterialForType(selectedNode?.data.outputs.type.toLowerCase() || ""), + queryFn: () => transformService.getFlowsRawMaterialForType(selectedNode?.data.outputs.type.toLowerCase() || ""), }) const [searchTerm, setSearchTerm] = useState("") @@ -54,7 +54,7 @@ const TransformSheet = ({ onLayout }: { onLayout: () => void }) => { const handleClick = useCallback((scanner: Scanner) => { if (!selectedNode) return const position = { x: selectedNode.position.x + 350, y: selectedNode.position.y } - const newNode: TransformNode = { + const newNode: FlowNode = { id: `${scanner.name}-${Date.now()}`, type: scanner.type === "type" ? "type" : "scanner", position, @@ -79,7 +79,7 @@ const TransformSheet = ({ onLayout }: { onLayout: () => void }) => { } setNodes((prev) => ([...prev, newNode])) - const connection: TransformEdge = { + const connection: FlowEdge = { id: `${selectedNode.id}-${newNode.id}`, source: selectedNode.id, target: newNode.id, @@ -88,12 +88,12 @@ const TransformSheet = ({ onLayout }: { onLayout: () => void }) => { } setEdges((prev) => ([...prev, connection])) // onLayout && onLayout() - setOpenTransformSheet(false) - }, [selectedNode, setNodes, setEdges, onLayout, setOpenTransformSheet]) + setOpenFlowSheet(false) + }, [selectedNode, setNodes, setEdges, onLayout, setOpenFlowSheet]) return (
- + Add connector to {selectedNode?.data.class_name} @@ -144,7 +144,7 @@ const TransformSheet = ({ onLayout }: { onLayout: () => void }) => { } -export default TransformSheet +export default FlowSheet // Custom equality function for ScannerItem function areEqual(prevProps: { scanner: Scanner }, nextProps: { scanner: Scanner }) { diff --git a/flowsint-app/src/renderer/src/components/transforms/new-transform.tsx b/flowsint-app/src/renderer/src/components/flows/new-flow.tsx similarity index 74% rename from flowsint-app/src/renderer/src/components/transforms/new-transform.tsx rename to flowsint-app/src/renderer/src/components/flows/new-flow.tsx index d161812..c9213f7 100644 --- a/flowsint-app/src/renderer/src/components/transforms/new-transform.tsx +++ b/flowsint-app/src/renderer/src/components/flows/new-flow.tsx @@ -2,12 +2,12 @@ import { transformService } from '@/api/transfrom-service' import { useCallback, ReactNode, cloneElement, isValidElement } from 'react' import { toast } from 'sonner' import { useNavigate } from '@tanstack/react-router' -import { useTransformStore } from '@/stores/transform-store' +import { useFlowStore } from '@/stores/flow-store' -const NewTransform = ({ children }: { children: ReactNode }) => { +const NewFlow = ({ children }: { children: ReactNode }) => { const navigate = useNavigate() - const setNodes = useTransformStore(state => state.setNodes) - const setEdges = useTransformStore(state => state.setEdges) + const setNodes = useFlowStore(state => state.setNodes) + const setEdges = useFlowStore(state => state.setEdges) const handleCreateTransform = useCallback(async () => { toast.promise( @@ -19,9 +19,9 @@ const NewTransform = ({ children }: { children: ReactNode }) => { name: "New flow", description: "A new example flow.", category: [], - transform_schema: {} + flow_schema: {} })) - navigate({ to: `/dashboard/transforms/${response.id}` }) + navigate({ to: `/dashboard/flows/${response.id}` }) return response })(), { @@ -41,4 +41,4 @@ const NewTransform = ({ children }: { children: ReactNode }) => { onClick: handleCreateTransform }) } -export default NewTransform \ No newline at end of file +export default NewFlow \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/transforms/params-dialog.tsx b/flowsint-app/src/renderer/src/components/flows/params-dialog.tsx similarity index 94% rename from flowsint-app/src/renderer/src/components/transforms/params-dialog.tsx rename to flowsint-app/src/renderer/src/components/flows/params-dialog.tsx index 2d0258c..cccf384 100644 --- a/flowsint-app/src/renderer/src/components/transforms/params-dialog.tsx +++ b/flowsint-app/src/renderer/src/components/flows/params-dialog.tsx @@ -1,4 +1,4 @@ -import { useTransformStore } from '@/stores/transform-store' +import { useFlowStore } from '@/stores/flow-store' import { DialogHeader, DialogFooter, Dialog, DialogContent, DialogTitle } from '../ui/dialog' import { Button } from '../ui/button' import { useCallback, useState, useEffect } from 'react' @@ -14,10 +14,10 @@ import { MemoizedMarkdown } from '../chat/memoized-markdown' import { cn } from '@/lib/utils' const ParamsDialog = () => { - const openParamsDialog = useTransformStore(s => s.openParamsDialog) - const setOpenParamsDialog = useTransformStore(s => s.setOpenParamsDialog) - const selectedNode = useTransformStore(s => s.selectedNode) - const updateNode = useTransformStore(s => s.updateNode) + const openParamsDialog = useFlowStore(s => s.openParamsDialog) + const setOpenParamsDialog = useFlowStore(s => s.setOpenParamsDialog) + const selectedNode = useFlowStore(s => s.selectedNode) + const updateNode = useFlowStore(s => s.updateNode) const [params, setParams] = useState>({}) const [settings, setSettings] = useState>({ duration: '30', diff --git a/flowsint-app/src/renderer/src/components/transforms/raw-material.tsx b/flowsint-app/src/renderer/src/components/flows/raw-material.tsx similarity index 96% rename from flowsint-app/src/renderer/src/components/transforms/raw-material.tsx rename to flowsint-app/src/renderer/src/components/flows/raw-material.tsx index 4efde11..03e8cf7 100644 --- a/flowsint-app/src/renderer/src/components/transforms/raw-material.tsx +++ b/flowsint-app/src/renderer/src/components/flows/raw-material.tsx @@ -4,16 +4,16 @@ import ScannerItem from "./scanner-item" import { type Scanner } from "@/types/transform" import { Input } from "../ui/input" import { Button } from "../ui/button" -import { transformService } from "@/api/transfrom-service" import { useQuery } from "@tanstack/react-query" import { SkeletonList } from "../shared/skeleton-list" +import { flowService } from "@/api/flow-service" export default function RawMaterial() { const { data: materials, isLoading, error } = useQuery({ queryKey: ["raw_material"], - queryFn: () => transformService.getRawMaterial(), + queryFn: () => flowService.getRawMaterial(), }) const [searchTerm, setSearchTerm] = useState("") diff --git a/flowsint-app/src/renderer/src/components/transforms/save-modal.tsx b/flowsint-app/src/renderer/src/components/flows/save-modal.tsx similarity index 85% rename from flowsint-app/src/renderer/src/components/transforms/save-modal.tsx rename to flowsint-app/src/renderer/src/components/flows/save-modal.tsx index 8ee561e..2c95ca8 100644 --- a/flowsint-app/src/renderer/src/components/transforms/save-modal.tsx +++ b/flowsint-app/src/renderer/src/components/flows/save-modal.tsx @@ -14,14 +14,14 @@ import { Label } from "@/components/ui/label" import { Textarea } from "@/components/ui/textarea" import { useKeyboardShortcut } from "@/hooks/use-keyboard-shortcut" -interface TransformModalProps { +interface SaveModalProps { open: boolean onOpenChange: (open: boolean) => void onSave: (name: string, description: string) => void isLoading: boolean } -export function TransformModal({ open, onOpenChange, onSave, isLoading }: TransformModalProps) { +export function SaveModal({ open, onOpenChange, onSave, isLoading }: SaveModalProps) { const [name, setName] = useState("My Transform") const [description, setDescription] = useState("") const [nameError, setNameError] = useState("") @@ -36,7 +36,7 @@ export function TransformModal({ open, onOpenChange, onSave, isLoading }: Transf const handleSave = () => { if (!name.trim()) { - setNameError("Transform name is required") + setNameError("Flow name is required") return } onSave(name, description) @@ -46,8 +46,8 @@ export function TransformModal({ open, onOpenChange, onSave, isLoading }: Transf - Save Transform - Give your transform a name and description before saving. + Save Flow + Give your flow a name and description before saving.
@@ -62,7 +62,7 @@ export function TransformModal({ open, onOpenChange, onSave, isLoading }: Transf setName(e.target.value) if (e.target.value.trim()) setNameError("") }} - placeholder="Enter transform name" + placeholder="Enter flow name" className={nameError ? "border-destructive" : ""} />
@@ -83,7 +83,7 @@ export function TransformModal({ open, onOpenChange, onSave, isLoading }: Transf Cancel diff --git a/flowsint-app/src/renderer/src/components/transforms/scanner-data.tsx b/flowsint-app/src/renderer/src/components/flows/scanner-data.tsx similarity index 52% rename from flowsint-app/src/renderer/src/components/transforms/scanner-data.tsx rename to flowsint-app/src/renderer/src/components/flows/scanner-data.tsx index 9e3ea9f..5ab64be 100644 --- a/flowsint-app/src/renderer/src/components/transforms/scanner-data.tsx +++ b/flowsint-app/src/renderer/src/components/flows/scanner-data.tsx @@ -16,6 +16,34 @@ export const categoryColors: Record = { organization: "#9C27B0", } +// Graph viewer specific colors +export const GRAPH_COLORS = { + // Link colors + LINK_DEFAULT: 'rgba(128, 128, 128, 0.6)', + LINK_HIGHLIGHTED: 'rgba(255, 115, 0, 0.68)', + LINK_DIMMED: 'rgba(133, 133, 133, 0.23)', + LINK_LABEL_HIGHLIGHTED: 'rgba(255, 115, 0, 0.9)', + LINK_LABEL_DEFAULT: 'rgba(180, 180, 180, 0.75)', + + // Node highlight colors + NODE_HIGHLIGHT_HOVER: 'rgba(255, 0, 0, 0.3)', + NODE_HIGHLIGHT_DEFAULT: 'rgba(255, 165, 0, 0.3)', + + // Text colors + TEXT_LIGHT: '#161616', + TEXT_DARK: '#FFFFFF', + + // Background colors + BACKGROUND_LIGHT: "#FFFFFF", + BACKGROUND_DARK: '#161616', + + // Transparent colors + TRANSPARENT: "#00000000", + + // Default node color + NODE_DEFAULT: '#0074D9', +} as const + /** * Get the color for a scanner based on its type or category * @param type The scanner type diff --git a/flowsint-app/src/renderer/src/components/transforms/scanner-item.tsx b/flowsint-app/src/renderer/src/components/flows/scanner-item.tsx similarity index 100% rename from flowsint-app/src/renderer/src/components/transforms/scanner-item.tsx rename to flowsint-app/src/renderer/src/components/flows/scanner-item.tsx diff --git a/flowsint-app/src/renderer/src/components/transforms/scanner-node.tsx b/flowsint-app/src/renderer/src/components/flows/scanner-node.tsx similarity index 96% rename from flowsint-app/src/renderer/src/components/transforms/scanner-node.tsx rename to flowsint-app/src/renderer/src/components/flows/scanner-node.tsx index d4f601a..9dd4019 100644 --- a/flowsint-app/src/renderer/src/components/transforms/scanner-node.tsx +++ b/flowsint-app/src/renderer/src/components/flows/scanner-node.tsx @@ -7,7 +7,7 @@ import { useNodesDisplaySettings } from "@/stores/node-display-settings" import { cn } from "@/lib/utils" import { Plus, TriangleAlert } from "lucide-react" import { type ScannerNodeProps } from "@/types/transform" -import { TransformNode, useTransformStore } from "@/stores/transform-store" +import { FlowNode, useFlowStore } from "@/stores/flow-store" import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip" import { Button } from "../ui/button" import { ButtonHandle } from "../xyflow/button-handle" @@ -50,12 +50,12 @@ const ScannerNode = memo(({ data, isConnectable }: ScannerNodeProps) => { const inputColor = colors[data.inputs.type.toLowerCase()] const outputColor = colors[data.outputs.type.toLowerCase()] const opacity = data.computationState === "pending" ? 0.5 : 1 - const setOpenTransformSheet = useTransformStore(state => state.setOpenTransformSheet) + const setOpenFlowSheet = useFlowStore(state => state.setOpenFlowSheet) const Icon = data.type === "type" ? useIcon(data.outputs.type.toLowerCase() as string, null) : data.icon ? useIcon(data.icon, null) : null const handleAddConnector = useCallback(() => { - setOpenTransformSheet(true, data as unknown as TransformNode) - }, [setOpenTransformSheet, data]) + setOpenFlowSheet(true, data as unknown as FlowNode) + }, [setOpenFlowSheet, data]) const getStatusVariant = (state?: string) => { switch (state) { diff --git a/flowsint-app/src/renderer/src/components/transforms/test-transform.tsx b/flowsint-app/src/renderer/src/components/flows/test-flow.tsx similarity index 100% rename from flowsint-app/src/renderer/src/components/transforms/test-transform.tsx rename to flowsint-app/src/renderer/src/components/flows/test-flow.tsx diff --git a/flowsint-app/src/renderer/src/components/transforms/transform-item.tsx b/flowsint-app/src/renderer/src/components/flows/transform-item.tsx similarity index 93% rename from flowsint-app/src/renderer/src/components/transforms/transform-item.tsx rename to flowsint-app/src/renderer/src/components/flows/transform-item.tsx index 25f5304..63857a8 100644 --- a/flowsint-app/src/renderer/src/components/transforms/transform-item.tsx +++ b/flowsint-app/src/renderer/src/components/flows/transform-item.tsx @@ -15,10 +15,10 @@ export const TransformItem = memo(({ transform }: { transform: any }) => { }) : null - const stepsCount = transform?.transform_schema?.edges?.length || 0 + const stepsCount = transform?.flow_schema?.edges?.length || 0 return ( - + diff --git a/flowsint-app/src/renderer/src/components/transforms/type-node.tsx b/flowsint-app/src/renderer/src/components/flows/type-node.tsx similarity index 92% rename from flowsint-app/src/renderer/src/components/transforms/type-node.tsx rename to flowsint-app/src/renderer/src/components/flows/type-node.tsx index 35b13ef..12c8c30 100644 --- a/flowsint-app/src/renderer/src/components/transforms/type-node.tsx +++ b/flowsint-app/src/renderer/src/components/flows/type-node.tsx @@ -15,7 +15,7 @@ import { NodeTooltipTrigger, } from "@/components/xyflow/node-tooltip"; import { BaseNode, BaseNodeContent } from "@/components/xyflow/base-node"; -import { TransformNode, useTransformStore } from "@/stores/transform-store" +import { FlowNode, useFlowStore } from "@/stores/flow-store" // Custom equality function to prevent unnecessary re-renders function areEqual(prevProps: ScannerNodeProps, nextProps: ScannerNodeProps) { @@ -32,12 +32,12 @@ const TypeNode = memo(({ data }: ScannerNodeProps) => { const colors = useNodesDisplaySettings(s => s.colors) const outputColor = colors[data.outputs.type.toLowerCase()] const Icon = useIcon(data.outputs.type.toLowerCase() as string, null) - const setOpenTransformSheet = useTransformStore(state => state.setOpenTransformSheet) + const setOpenFlowSheet = useFlowStore(state => state.setOpenFlowSheet) const key = data.outputs.properties[0].name const handleAddConnector = useCallback(() => { - setOpenTransformSheet(true, data as unknown as TransformNode) - }, [setOpenTransformSheet, data]) + setOpenFlowSheet(true, data as unknown as FlowNode) + }, [setOpenFlowSheet, data]) return ( diff --git a/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx index 993ee05..53c8864 100644 --- a/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx @@ -1,12 +1,15 @@ import React, { memo, useCallback, useState } from 'react'; import { transformService } from '@/api/transfrom-service'; +import { flowService } from '@/api/flow-service'; import { useQuery } from '@tanstack/react-query'; import { Skeleton } from '@/components/ui/skeleton'; import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; -import { FileCode2, Search, Info, Star } from 'lucide-react'; -import { Transform } from '@/types'; +import { Tabs, TabsList, TabsTrigger, TabsContent } from '@/components/ui/tabs'; +import { FileCode2, Search, Info, Star, Zap } from 'lucide-react'; +import { Transform, Flow } from '@/types'; import { GraphNode } from '@/stores/graph-store'; +import { useLaunchFlow } from '@/hooks/use-launch-flow'; import { useLaunchTransform } from '@/hooks/use-launch-transform'; import { useParams } from '@tanstack/react-router'; import { capitalizeFirstLetter, cn } from '@/lib/utils'; @@ -43,25 +46,47 @@ export default function ContextMenu({ ...props }: GraphContextMenuProps) { const { id: sketchId } = useParams({ strict: false }) - const [searchQuery, setSearchQuery] = useState(''); + const [activeTab, setActiveTab] = useState('transforms'); + const [transformsSearchQuery, setTransformsSearchQuery] = useState(''); + const [flowsSearchQuery, setFlowsSearchQuery] = useState(''); + const { launchFlow } = useLaunchFlow(false); const { launchTransform } = useLaunchTransform(false); - const { data: transforms, isLoading } = useQuery({ + const { data: transforms, isLoading: isLoadingTransforms } = useQuery({ queryKey: ["transforms", node.data.type], queryFn: () => transformService.get(capitalizeFirstLetter(node.data.type)), }); + const { data: flows, isLoading: isLoadingFlows } = useQuery({ + queryKey: ["flows", node.data.type], + queryFn: () => flowService.get(capitalizeFirstLetter(node.data.type)), + }); + const filteredTransforms = transforms?.filter((transform: Transform) => { - if (!searchQuery.trim()) return true; - const query = searchQuery.toLowerCase().trim(); + if (!transformsSearchQuery.trim()) return true; + const query = transformsSearchQuery.toLowerCase().trim(); const matchesName = transform.name?.toLowerCase().includes(query); const matchesDescription = transform.description?.toLowerCase().includes(query); return matchesName || matchesDescription; }) || []; - const handleTransformClick = (e: React.MouseEvent, transformId: string) => { + const filteredFlows = flows?.filter((flow: Flow) => { + if (!flowsSearchQuery.trim()) return true; + const query = flowsSearchQuery.toLowerCase().trim(); + const matchesName = flow.name?.toLowerCase().includes(query); + const matchesDescription = flow.description?.toLowerCase().includes(query); + return matchesName || matchesDescription; + }) || []; + + const handleFlowClick = (e: React.MouseEvent, flowId: string) => { e.stopPropagation(); - launchTransform([node.data.label], transformId, sketchId) + launchFlow([node.data.label], flowId, sketchId) + setMenu(null) + }; + + const handleTransformClick = (e: React.MouseEvent, transformName: string) => { + e.stopPropagation(); + launchTransform([node.data.label], transformName, sketchId) setMenu(null) }; @@ -84,75 +109,176 @@ export default function ContextMenu({
- {/* Search bar */} + {/* Tabs */}
-
- - { - e.stopPropagation(); - setSearchQuery(e.target.value); - }} - onClick={(e) => e.stopPropagation()} - className="h-7 pl-7 text-xs" - /> -
+ + + e.stopPropagation()} + > + + Transforms + + e.stopPropagation()} + > + + Flows + + +
- {/* Transforms list */} -
- {isLoading ? ( -
- {[...Array(3)].map((_, i) => ( -
- -
- - -
+ {/* Tab Content */} + + {/* Transforms Tab */} + + {/* Transforms Search */} +
+
+ + { + e.stopPropagation(); + setTransformsSearchQuery(e.target.value); + }} + onClick={(e) => e.stopPropagation()} + className="h-7 pl-7 text-xs" + /> +
+
+ + {/* Transforms List */} +
+ {isLoadingTransforms ? ( +
+ {[...Array(3)].map((_, i) => ( +
+ +
+ + +
+
+ ))}
- ))} + ) : filteredTransforms.length > 0 ? ( +
+ {filteredTransforms.map((transform: Transform) => ( + +
+ + ))} +
+ ) : ( +
+

+ {transformsSearchQuery ? 'No transforms found' : 'No transforms available'} +

+
+ )}
- ) : filteredTransforms.length > 0 ? ( -
- {filteredTransforms.map((transform: Transform) => ( - -
- - ))} + {/* Flows Tab */} + + {/* Flows Search */} +
+
+ + { + e.stopPropagation(); + setFlowsSearchQuery(e.target.value); + }} + onClick={(e) => e.stopPropagation()} + className="h-7 pl-7 text-xs" + /> +
- ) : ( -
-

- {searchQuery ? 'No transforms found' : 'No transforms available'} -

+ + {/* Flows List */} +
+ {isLoadingFlows ? ( +
+ {[...Array(3)].map((_, i) => ( +
+ +
+ + +
+
+ ))} +
+ ) : filteredFlows.length > 0 ? ( +
+ {filteredFlows.map((flow: Flow) => ( + +
+ + ))} +
+ ) : ( +
+

+ {flowsSearchQuery ? 'No flows found' : 'No flows available'} +

+
+ )}
- )} -
+ + ); } diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx index e5e37a5..109bba8 100644 --- a/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/details-panel.tsx @@ -2,7 +2,7 @@ import { memo } from "react" import { cn } from "@/lib/utils" import { CopyButton } from "@/components/copy" import { Check, Rocket, X, MousePointer } from "lucide-react" -import LaunchTransform from "../launch-transform" +import LaunchFlow from "../launch-transform" import NodeActions from "../node-actions" import { GraphNode } from "@/stores/graph-store" import { Button } from "../../ui/button" @@ -33,7 +33,7 @@ export default function DetailsPanel({ node }: { node: GraphNode | null }) { return (
- + {/* */}

{node.data?.label}

@@ -42,7 +42,7 @@ export default function DetailsPanel({ node }: { node: GraphNode | null }) {
- + - +
diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel/relationships.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel/relationships.tsx index b8d33bd..b6524ff 100644 --- a/flowsint-app/src/renderer/src/components/graphs/details-panel/relationships.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/details-panel/relationships.tsx @@ -9,6 +9,7 @@ import { ArrowRight } from 'lucide-react'; import { memo, useCallback, useRef } from 'react'; type Relation = { + source: GraphNode, target: GraphNode, edge: { label: string } } @@ -16,9 +17,10 @@ type Relation = { const getInlineRelationships = (nodes: GraphNode[], edges: GraphEdge[]): Relation[] => { const relationships: Relation[] = [] edges.forEach((edge) => { + const source = nodes.find((n) => (n.id === edge.source)) const target = nodes.find((n) => (n.id === edge.target)) - if (!target) return - relationships.push({ target, edge: { label: String(edge.caption || 'RELATED_TO') } }) + if (!target || !source) return + relationships.push({ source, target, edge: { label: String(edge.caption || 'RELATED_TO') } }) }) return relationships } @@ -66,7 +68,9 @@ const Relationships = ({ sketchId, nodeId }: { sketchId: string, nodeId: string className="mb-1 px-3" > - {rel.edge.label} + + + {rel.edge.label} @@ -87,8 +91,8 @@ const RelationshipItem = memo(({ node }: { node: GraphNode }) => { setCurrentNode(node) }, [setCurrentNode]) return ( - ) }) \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx index f0364b7..a87d343 100644 --- a/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx @@ -6,7 +6,7 @@ import ContextMenu from './context-menu' const GraphMain = () => { const nodes = useGraphStore(s => s.nodes) const edges = useGraphStore(s => s.edges) - // const currentNode = useGraphStore(s => s.currentNode) + const currentNode = useGraphStore(s => s.currentNode) const toggleNodeSelection = useGraphStore(s => s.toggleNodeSelection) const clearSelectedNodes = useGraphStore(s => s.clearSelectedNodes) @@ -14,12 +14,12 @@ const GraphMain = () => { const containerRef = useRef(null) const [menu, setMenu] = React.useState(null) - // // Handle current node centering - // useEffect(() => { - // if (!currentNode || !graphRef.current) return - // graphRef.current.centerAt(currentNode.x, currentNode.y, 1000) - // graphRef.current.zoom(5, 2000) - // }, [currentNode]) + // Handle current node centering + useEffect(() => { + if (!currentNode || !graphRef.current) return + graphRef.current.centerAt(currentNode.x, currentNode.y, 500) + graphRef.current.zoom(5, 500) + }, [currentNode]) const handleNodeClick = useCallback((node: any) => { toggleNodeSelection(node, false) diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx index bb6b60f..7838f01 100644 --- a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx @@ -21,7 +21,7 @@ const RelationshipsTable = lazy(() => import('@/components/table/relationships-v const Graph = lazy(() => import('./graph')) // const Wall = lazy(() => import('./wall/wall')) -const NODE_COUNT_THRESHOLD = 500; +const NODE_COUNT_THRESHOLD = 4500; // Separate component for the drag overlay const DragOverlay = memo(({ isDragging }: { isDragging: boolean }) => ( diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx index c4cacf4..86c02e4 100644 --- a/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx @@ -7,6 +7,7 @@ import React, { useCallback, useMemo, useEffect, useState, useRef } from 'react' import ForceGraph2D from 'react-force-graph-2d'; import { Button } from '../ui/button'; import { useTheme } from '@/components/theme-provider' +import { GRAPH_COLORS } from '../flows/scanner-data'; interface GraphViewerProps { nodes: GraphNode[]; @@ -26,9 +27,10 @@ interface GraphViewerProps { onGraphRef?: (ref: any) => void; } -const NODE_COUNT_THRESHOLD = 500; const CONSTANTS = { - NODE_DEFAULT_SIZE: 24, + NODE_COUNT_THRESHOLD: 500, + NODE_DEFAULT_SIZE: 10, + NODE_LABEL_FONT_SIZE: 3.5, LABEL_FONT_SIZE: 2.5, NODE_FONT_SIZE: 5, LABEL_NODE_MARGIN: 18, @@ -37,7 +39,6 @@ const CONSTANTS = { PI: Math.PI, MEASURE_FONT: '1px Sans-Serif', MIN_FONT_SIZE: 0.5, - LINK_COLOR: 'rgba(128, 128, 128, 0.6)', LINK_WIDTH: 1, ARROW_SIZE: 8, ARROW_ANGLE: Math.PI / 6 @@ -50,6 +51,43 @@ const LABEL_FONT_STRING = `${CONSTANTS.LABEL_FONT_SIZE}px Sans-Serif`; const tempPos = { x: 0, y: 0 }; const tempDimensions = [0, 0]; +// Image cache for icons +const imageCache = new Map(); +const imageLoadPromises = new Map>(); + +// Preload icon images +const preloadImage = (iconType: string): Promise => { + const cacheKey = iconType; + + // Return cached image if available + if (imageCache.has(cacheKey)) { + return Promise.resolve(imageCache.get(cacheKey)!); + } + + // Return existing promise if already loading + if (imageLoadPromises.has(cacheKey)) { + return imageLoadPromises.get(cacheKey)!; + } + + // Create new loading promise + const promise = new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => { + imageCache.set(cacheKey, img); + imageLoadPromises.delete(cacheKey); + resolve(img); + }; + img.onerror = () => { + imageLoadPromises.delete(cacheKey); + reject(new Error(`Failed to load icon: ${iconType}`)); + }; + img.src = `/icons/${iconType}.svg`; + }); + + imageLoadPromises.set(cacheKey, promise); + return promise; +}; + const GraphViewer: React.FC = ({ nodes, edges, @@ -68,10 +106,17 @@ const GraphViewer: React.FC = ({ const [currentZoom, setCurrentZoom] = useState(1); const [dimensions, setDimensions] = useState({ width: 0, height: 0 }); + // Hover highlighting state + const [highlightNodes, setHighlightNodes] = useState>(new Set()); + const [highlightLinks, setHighlightLinks] = useState>(new Set()); + const [hoverNode, setHoverNode] = useState(null); + // Store references const containerRef = useRef(null); const graphRef = useRef(); const isGraphReadyRef = useRef(false); + const lastRenderTimeRef = useRef(0); + const renderThrottleRef = useRef(null); // Store selectors const nodeColors = useNodesDisplaySettings(s => s.colors); @@ -82,6 +127,16 @@ const GraphViewer: React.FC = ({ const { theme } = useTheme(); const setOpenMainDialog = useGraphStore(state => state.setOpenMainDialog); + // Preload icons when nodes change + useEffect(() => { + if (showIcons) { + const iconTypes = new Set(nodes.map(node => node.data?.type as ItemType).filter(Boolean)); + iconTypes.forEach(type => { + preloadImage(type).catch(console.warn); // Silently handle failures + }); + } + }, [nodes, showIcons]); + // Optimized graph initialization callback const initializeGraph = useCallback((graphInstance: any) => { if (!graphInstance || isGraphReadyRef.current) return; @@ -114,10 +169,10 @@ const GraphViewer: React.FC = ({ // Memoized rendering check const shouldUseSimpleRendering = useMemo(() => - nodes.length > NODE_COUNT_THRESHOLD || currentZoom < 2.5 + nodes.length > CONSTANTS.NODE_COUNT_THRESHOLD || currentZoom < 2.5 , [nodes.length, currentZoom]); - // Optimized graph data transformation + // Optimized graph data transformation with proper memoization dependencies const graphData = useMemo(() => { // Transform nodes const transformedNodes = nodes.map(node => { @@ -125,13 +180,18 @@ const GraphViewer: React.FC = ({ return { ...node, nodeLabel: node.data?.label || node.id, - nodeColor: nodeColors[type] || '#0074D9', + nodeColor: nodeColors[type] || GRAPH_COLORS.NODE_DEFAULT, nodeSize: CONSTANTS.NODE_DEFAULT_SIZE, nodeType: type, val: getSize(type), + neighbors: [] as any[], + links: [] as any[] }; }); + // Create a map for quick node lookup + const nodeMap = new Map(transformedNodes.map(node => [node.id, node])); + // Group and transform edges const edgeGroups = new Map(); edges.forEach(edge => { @@ -158,6 +218,26 @@ const GraphViewer: React.FC = ({ }; }); + // Build node relationships (neighbors and links) + transformedEdges.forEach(link => { + const sourceNode = nodeMap.get(link.source); + const targetNode = nodeMap.get(link.target); + + if (sourceNode && targetNode) { + // Add neighbors + if (!sourceNode.neighbors.includes(targetNode)) { + sourceNode.neighbors.push(targetNode); + } + if (!targetNode.neighbors.includes(sourceNode)) { + targetNode.neighbors.push(sourceNode); + } + + // Add links + sourceNode.links.push(link); + targetNode.links.push(link); + } + }); + // Handle hierarchy layout if (view === "hierarchy") { const { nodes: nds, edges: eds } = getDagreLayoutedElements(transformedNodes, transformedEdges); @@ -173,66 +253,256 @@ const GraphViewer: React.FC = ({ }; }, [nodes, edges, nodeColors, getSize, view]); - // Event handlers - const handleNodeClick = useCallback((node: any) => onNodeClick?.(node), [onNodeClick]); - const handleNodeRightClick = useCallback((node: any, event: MouseEvent) => onNodeRightClick?.(node, event), [onNodeRightClick]); - const handleBackgroundClick = useCallback(() => onBackgroundClick?.(), [onBackgroundClick]); - const handleOpenNewAddItemDialog = useCallback(() => setOpenMainDialog(true), [setOpenMainDialog]); + // Event handlers with proper memoization + const handleNodeClick = useCallback((node: any) => { + onNodeClick?.(node); + }, [onNodeClick]); - // Optimized node rendering + const handleNodeRightClick = useCallback((node: any, event: MouseEvent) => { + onNodeRightClick?.(node, event); + }, [onNodeRightClick]); + + const handleBackgroundClick = useCallback(() => { + onBackgroundClick?.(); + }, [onBackgroundClick]); + + const handleOpenNewAddItemDialog = useCallback(() => { + setOpenMainDialog(true); + }, [setOpenMainDialog]); + + // Throttled hover handlers to reduce excessive re-renders + const handleNodeHover = useCallback((node: any) => { + // Throttle hover updates to max 60fps + const now = Date.now(); + if (now - lastRenderTimeRef.current < 16) { // ~60fps + if (renderThrottleRef.current) { + clearTimeout(renderThrottleRef.current); + } + renderThrottleRef.current = setTimeout(() => { + handleNodeHover(node); + }, 16) as any; + return; + } + lastRenderTimeRef.current = now; + + const newHighlightNodes = new Set(); + const newHighlightLinks = new Set(); + + if (node) { + // Add the hovered node + newHighlightNodes.add(node.id); + + // Add connected nodes and links + if (node.neighbors) { + node.neighbors.forEach((neighbor: any) => { + newHighlightNodes.add(neighbor.id); + }); + } + + if (node.links) { + node.links.forEach((link: any) => { + newHighlightLinks.add(`${link.source.id}-${link.target.id}`); + }); + } + + setHoverNode(node.id); + } else { + setHoverNode(null); + } + + setHighlightNodes(newHighlightNodes); + setHighlightLinks(newHighlightLinks); + }, []); + + const handleLinkHover = useCallback((link: any) => { + // Throttle hover updates to max 60fps + const now = Date.now(); + if (now - lastRenderTimeRef.current < 16) { // ~60fps + if (renderThrottleRef.current) { + clearTimeout(renderThrottleRef.current); + } + renderThrottleRef.current = setTimeout(() => { + handleLinkHover(link); + }, 16) as any; + return; + } + lastRenderTimeRef.current = now; + + const newHighlightNodes = new Set(); + const newHighlightLinks = new Set(); + + if (link) { + // Add the hovered link + newHighlightLinks.add(`${link.source}-${link.target}`); + + // Add connected nodes + newHighlightNodes.add(link.source.id); + newHighlightNodes.add(link.target.id); + } + + setHoverNode(null); + setHighlightNodes(newHighlightNodes); + setHighlightLinks(newHighlightLinks); + }, []); + + // Optimized node rendering with proper icon caching const renderNode = useCallback((node: any, ctx: CanvasRenderingContext2D, globalScale: number) => { - const size = node.nodeSize * (settings.nodeSize.value / 50); + const size = node.nodeSize * (settings.nodeSize.value / 100 + .4); + const isHighlighted = highlightNodes.has(node.id); + const hasAnyHighlight = highlightNodes.size > 0 || highlightLinks.size > 0; + const isHovered = hoverNode === node.id; + + // Draw highlight ring for highlighted nodes + if (isHighlighted) { + ctx.beginPath(); + ctx.arc(node.x, node.y, size * 1.2, 0, 2 * Math.PI); + ctx.fillStyle = isHovered ? GRAPH_COLORS.NODE_HIGHLIGHT_HOVER : GRAPH_COLORS.NODE_HIGHLIGHT_DEFAULT; + ctx.fill(); + } + + // Set node color based on highlight state + if (hasAnyHighlight) { + ctx.fillStyle = isHighlighted ? node.nodeColor : `${node.nodeColor}7D`; + } else { + ctx.fillStyle = node.nodeColor; + } - // Always draw the basic circle ctx.beginPath(); - ctx.arc(node.x, node.y, size * 0.65, 0, 2 * Math.PI); - ctx.fillStyle = node.nodeColor; + ctx.arc(node.x, node.y, size, 0, 2 * Math.PI); ctx.fill(); // Early exit for simple rendering if (shouldUseSimpleRendering) return; - // Icon rendering - if (showIcons) { - const img = new Image(); - img.src = `/icons/${node.nodeType}.svg`; - ctx.drawImage(img, node.x - size / 2, node.y - size / 2, size, size); + // Optimized icon rendering with cached images + if (showIcons && node.nodeType) { + const cachedImage = imageCache.get(node.nodeType); + if (cachedImage && cachedImage.complete) { + try { + ctx.drawImage(cachedImage, node.x - size / 2, node.y - size / 2, size, size); + } catch (error) { + // Silently handle drawing errors + } + } } - // Label rendering + // Optimized label rendering if (showLabels && globalScale > 3) { const label = node.nodeLabel || node.label || node.id; if (label) { - const fontSize = CONSTANTS.NODE_FONT_SIZE * (size / 10); - ctx.font = `${fontSize}px Sans-Serif` - const bgHeight = CONSTANTS.NODE_FONT_SIZE + 2; + // Only show labels for highlighted nodes when there's any highlighting + // or show all labels when there's no highlighting + if (hasAnyHighlight && !isHighlighted) { + return; + } + + const fontSize = Math.max(CONSTANTS.MIN_FONT_SIZE, CONSTANTS.NODE_FONT_SIZE * (size / 7)); + ctx.font = `${fontSize}px Sans-Serif`; + + const bgHeight = fontSize + 2; const bgY = node.y + size / 2 + 1; - ctx.fillStyle = theme === "light" ? '#161616' : '#FFFFFF'; + const color = theme === "light" ? GRAPH_COLORS.TEXT_LIGHT : GRAPH_COLORS.TEXT_DARK; + ctx.textAlign = 'center'; ctx.textBaseline = 'middle'; + ctx.fillStyle = isHighlighted ? color : `${color}2D`; ctx.fillText(label, node.x, bgY + bgHeight / 2); } } - }, [shouldUseSimpleRendering, showLabels, showIcons, settings.nodeSize.value, theme]); + }, [shouldUseSimpleRendering, showLabels, showIcons, settings.nodeSize.value, theme, highlightNodes, highlightLinks, hoverNode]); - // Optimized link rendering + // Optimized link rendering with reduced canvas state changes const linkCanvasObject = useCallback((link: any, ctx: CanvasRenderingContext2D) => { const { source: start, target: end } = link; - // Early exit for unbound links if (typeof start !== 'object' || typeof end !== 'object') return; + const linkKey = `${start.id}-${end.id}`; + const isHighlighted = highlightLinks.has(linkKey); + const hasAnyHighlight = highlightNodes.size > 0 || highlightLinks.size > 0; + + // Determine colors and styles once + let strokeStyle: string; + let lineWidth: number; + let fillStyle: string; + + if (isHighlighted) { + strokeStyle = GRAPH_COLORS.LINK_HIGHLIGHTED; + fillStyle = GRAPH_COLORS.LINK_HIGHLIGHTED; + lineWidth = CONSTANTS.LINK_WIDTH * (settings.linkWidth.value / 3); + } else if (hasAnyHighlight) { + strokeStyle = GRAPH_COLORS.LINK_DIMMED; + fillStyle = GRAPH_COLORS.LINK_DIMMED; + lineWidth = CONSTANTS.LINK_WIDTH * (settings.linkWidth.value / 5); + } else { + strokeStyle = GRAPH_COLORS.LINK_DEFAULT; + fillStyle = GRAPH_COLORS.LINK_DEFAULT; + lineWidth = CONSTANTS.LINK_WIDTH * (settings.linkWidth.value / 5); + } + // Draw connection line ctx.beginPath(); ctx.moveTo(start.x, start.y); ctx.lineTo(end.x, end.y); - ctx.strokeStyle = CONSTANTS.LINK_COLOR; - ctx.lineWidth = CONSTANTS.LINK_WIDTH * (settings.linkWidth.value / 5); + ctx.strokeStyle = strokeStyle; + ctx.lineWidth = lineWidth; ctx.stroke(); + // Draw directional arrow + const arrowLength = settings.linkDirectionalArrowLength?.value; + if (arrowLength && arrowLength > 0) { + const arrowRelPos = settings.linkDirectionalArrowRelPos?.value || 1; + + // Calculate arrow position along the link + let arrowX = start.x + (end.x - start.x) * arrowRelPos; + let arrowY = start.y + (end.y - start.y) * arrowRelPos; + + // If arrow is at the target node (arrowRelPos = 1), offset it to be at the node's edge + if (arrowRelPos === 1) { + const dx = end.x - start.x; + const dy = end.y - start.y; + const distance = Math.sqrt(dx * dx + dy * dy); + + if (distance > 0) { + // Calculate target node size (same as in renderNode function) + const targetNodeSize = (end.nodeSize || CONSTANTS.NODE_DEFAULT_SIZE) * (settings.nodeSize.value / 100 + 0.4); + + // Calculate offset to place arrow at node edge + const offset = targetNodeSize / distance; + arrowX = end.x - dx * offset; + arrowY = end.y - dy * offset; + } + } + + // Calculate arrow direction + const dx = end.x - start.x; + const dy = end.y - start.y; + const angle = Math.atan2(dy, dx); + + // Draw arrow head + ctx.save(); + ctx.translate(arrowX, arrowY); + ctx.rotate(angle); + + ctx.beginPath(); + ctx.moveTo(0, 0); + ctx.lineTo(-arrowLength, -arrowLength * 0.5); + ctx.lineTo(-arrowLength, arrowLength * 0.5); + ctx.closePath(); + + ctx.fillStyle = fillStyle; + ctx.fill(); + ctx.restore(); + } + // Early exit for simple rendering or no label if (shouldUseSimpleRendering || !link.label) return; + // Only show labels for highlighted links when there's any highlighting + if (hasAnyHighlight && !isHighlighted) { + return; + } + // Calculate label position and angle tempPos.x = (start.x + end.x) * 0.5; tempPos.y = (start.y + end.y) * 0.5; @@ -263,38 +533,69 @@ const GraphViewer: React.FC = ({ ctx.rotate(textAngle); // Background - ctx.fillStyle = theme === "light" ? "#FFFFFF" : '#161616'; + ctx.fillStyle = theme === "light" ? GRAPH_COLORS.BACKGROUND_LIGHT : GRAPH_COLORS.BACKGROUND_DARK; ctx.fillRect(-halfWidth, -halfHeight, tempDimensions[0], tempDimensions[1]); - // Text - ctx.fillStyle = 'darkgrey'; + // Text - follow same highlighting behavior as links + ctx.fillStyle = isHighlighted ? GRAPH_COLORS.LINK_LABEL_HIGHLIGHTED : GRAPH_COLORS.LINK_LABEL_DEFAULT; ctx.textAlign = 'center'; ctx.textBaseline = 'middle'; ctx.fillText(link.label, 0, 0); ctx.restore(); - }, [shouldUseSimpleRendering, settings.linkWidth.value, theme]); + }, [shouldUseSimpleRendering, settings.linkWidth.value, settings.linkDirectionalArrowLength?.value, settings.linkDirectionalArrowRelPos?.value, settings.nodeSize.value, theme, highlightLinks, highlightNodes]); - // Container resize observer + // Container resize observer with debouncing useEffect(() => { if (!containerRef.current) return; + let resizeTimeout: number; const resizeObserver = new ResizeObserver(entries => { - const { width: w, height: h } = entries[0].contentRect; - setDimensions({ width: w, height: h }); + // Debounce resize events + clearTimeout(resizeTimeout); + resizeTimeout = setTimeout(() => { + const { width: w, height: h } = entries[0].contentRect; + setDimensions({ width: w, height: h }); + }, 16) as any; // ~60fps }); resizeObserver.observe(containerRef.current); - return () => resizeObserver.disconnect(); + return () => { + resizeObserver.disconnect(); + clearTimeout(resizeTimeout); + }; }, []); - // Restart simulation when settings change + // Restart simulation when settings change (debounced) useEffect(() => { + let settingsTimeout: number | undefined; if (graphRef.current && isGraphReadyRef.current) { - graphRef.current.d3ReheatSimulation(); + if (settingsTimeout) clearTimeout(settingsTimeout); + settingsTimeout = setTimeout(() => { + graphRef.current?.d3ReheatSimulation(); + }, 100) as any; // Debounce settings changes } + return () => { + if (settingsTimeout) clearTimeout(settingsTimeout); + }; }, [settings]); + // Clear highlights when graph data changes + useEffect(() => { + setHighlightNodes(new Set()); + setHighlightLinks(new Set()); + setHoverNode(null); + }, [nodes, edges]); + + // Cleanup throttle timeouts + useEffect(() => { + return () => { + if (renderThrottleRef.current) { + clearTimeout(renderThrottleRef.current); + } + }; + }, []); + // Empty state if (!nodes.length) { return ( @@ -355,17 +656,13 @@ const GraphViewer: React.FC = ({ height={dimensions.height || height} graphData={graphData} nodeLabel="label" - nodeColor={node => shouldUseSimpleRendering ? node.nodeColor : "#00000000"} + nodeColor={node => shouldUseSimpleRendering ? node.nodeColor : GRAPH_COLORS.TRANSPARENT} nodeRelSize={6} onNodeRightClick={handleNodeRightClick} onNodeClick={handleNodeClick} onBackgroundClick={handleBackgroundClick} linkCurvature={link => link.curve} - linkDirectionalParticles={link => link.__highlighted ? 2 : 0} nodeCanvasObject={renderNode} - onNodeHover={(node => { - - })} onNodeDragEnd={(node => { node.fx = node.x; node.fy = node.y; @@ -377,13 +674,13 @@ const GraphViewer: React.FC = ({ d3VelocityDecay={settings.d3VelocityDecay?.value} warmupTicks={settings.warmupTicks?.value} dagLevelDistance={settings.dagLevelDistance?.value} - linkDirectionalArrowRelPos={settings.linkDirectionalArrowRelPos?.value} - linkDirectionalArrowLength={settings.linkDirectionalArrowLength?.value} - linkDirectionalParticleSpeed={settings.linkDirectionalParticleSpeed?.value} backgroundColor={backgroundColor} onZoom={(zoom) => setCurrentZoom(zoom.k)} linkCanvasObject={linkCanvasObject} enableNodeDrag={!shouldUseSimpleRendering} + autoPauseRedraw={false} + onNodeHover={handleNodeHover} + onLinkHover={handleLinkHover} />
); diff --git a/flowsint-app/src/renderer/src/components/graphs/launch-transform.tsx b/flowsint-app/src/renderer/src/components/graphs/launch-transform.tsx index 61eab43..558efc7 100644 --- a/flowsint-app/src/renderer/src/components/graphs/launch-transform.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/launch-transform.tsx @@ -12,49 +12,77 @@ import { } from "@/components/ui/sheet" import { Card, CardHeader, CardTitle, CardDescription } from "@/components/ui/card" import { RadioGroup, RadioGroupItem } from "@/components/ui/radio-group" +import { Tabs, TabsList, TabsTrigger, TabsContent } from "@/components/ui/tabs" +import { Input } from "@/components/ui/input" import { useLaunchTransform } from "@/hooks/use-launch-transform" +import { useLaunchFlow } from "@/hooks/use-launch-flow" import { formatDistanceToNow } from "date-fns" import { useQuery } from "@tanstack/react-query" import { transformService } from "@/api/transfrom-service" +import { flowService } from '@/api/flow-service'; import { useParams } from "@tanstack/react-router" import { capitalizeFirstLetter } from "@/lib/utils" import { Skeleton } from "@/components/ui/skeleton" +import { Search, FileCode2, Zap } from "lucide-react" +import { Transform, Flow } from "@/types" -interface Transform { - id: string - name: string - description: string - category: string - created_at: string - last_updated_at: string -} - -const LaunchTransform = ({ values, type, children }: { values: string[], type: string, children?: React.ReactNode }) => { +const LaunchTransformOrFlowPanel = ({ values, type, children }: { values: string[], type: string, children?: React.ReactNode }) => { const { launchTransform } = useLaunchTransform() + const { launchFlow } = useLaunchFlow() const { id: sketch_id } = useParams({ strict: false }) const [isOpen, setIsOpen] = useState(false) - const [selectedTransform, setSelectedTransform] = useState(null) + const [selectedTransform, setSelectedTransform] = useState(null) + const [activeTab, setActiveTab] = useState('transforms') + const [transformsSearchQuery, setTransformsSearchQuery] = useState('') + const [flowsSearchQuery, setFlowsSearchQuery] = useState('') - const { data: transforms, isLoading } = useQuery({ + const { data: transforms, isLoading: isLoadingTransforms } = useQuery({ queryKey: ["transforms", type], queryFn: () => transformService.get(capitalizeFirstLetter(type)), - // queryFn: () => transformService.get(), }); + const { data: flows, isLoading: isLoadingFlows } = useQuery({ + queryKey: ["flows", type], + queryFn: () => flowService.get(capitalizeFirstLetter(type)), + }); + + const filteredTransforms = transforms?.filter((transform: Transform) => { + if (!transformsSearchQuery.trim()) return true; + const query = transformsSearchQuery.toLowerCase().trim(); + const matchesName = transform.name?.toLowerCase().includes(query); + const matchesDescription = transform.description?.toLowerCase().includes(query); + return matchesName || matchesDescription; + }) || []; + + const filteredFlows = flows?.filter((transform: Flow) => { + if (!flowsSearchQuery.trim()) return true; + const query = flowsSearchQuery.toLowerCase().trim(); + const matchesName = transform.name?.toLowerCase().includes(query); + const matchesDescription = transform.description?.toLowerCase().includes(query); + return matchesName || matchesDescription; + }) || []; + const handleCloseModal = useCallback(() => { setIsOpen(false) }, []) - const handleSelectTransform = useCallback((transform: Transform) => { + const handleSelectTransform = useCallback((transform: Transform | Flow) => { setSelectedTransform(transform) }, []) - const handleLaunchTransform = useCallback(() => { + const handleLaunchPanel = useCallback(() => { if (selectedTransform) { - launchTransform(values, selectedTransform.id, sketch_id) + // Check if it's a Transform or Flow based on the active tab + if (activeTab === 'transforms') { + // For transforms, use name + launchTransform(values, (selectedTransform as Transform).name, sketch_id) + } else { + // For flows, use id + launchFlow(values, (selectedTransform as Flow).id, sketch_id) + } handleCloseModal() } - }, [selectedTransform, launchTransform, values, sketch_id]) + }, [selectedTransform, activeTab, launchTransform, launchFlow, values, sketch_id]) return (
@@ -68,71 +96,200 @@ const LaunchTransform = ({ values, type, children }: { values: string[], type: s Choose a transform to launch from the list below. -
- - {isLoading ? ( - // Skeleton loading state - Array.from({ length: 3 }).map((_, index) => ( - - -
-
- - -
-
- - -
-
- - -
-
- - - )) - ) : ( - transforms?.map((transform: Transform) => ( - handleSelectTransform(transform)} - > - -
-
- - {transform.name} -
- - {transform.description && ( - - {transform.description || "No description available"} - - )} - -
-
Created {formatDistanceToNow(transform.created_at, { addSuffix: true })}
-
Updated {formatDistanceToNow(transform.last_updated_at, { addSuffix: true })}
-
-
- - - )) - )} - + {/* Tabs */} +
+ + + e.stopPropagation()} + > + + Transforms + + e.stopPropagation()} + > + + Flows + + +
+ {/* Tab Content */} + + {/* Transforms Tab */} + + {/* Transforms Search */} +
+
+ + setTransformsSearchQuery(e.target.value)} + className="h-8 pl-7 text-sm" + /> +
+
+ + {/* Transforms List */} +
+ + {isLoadingTransforms ? ( + // Skeleton loading state + Array.from({ length: 3 }).map((_, index) => ( + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + )) + ) : filteredTransforms.length > 0 ? ( + filteredTransforms.map((transform: Transform) => ( + handleSelectTransform(transform)} + > + +
+
+ + {transform.name} +
+ + {transform.description && ( + + {transform.description || "No description available"} + + )} +
+ + + )) + ) : ( +
+

+ {transformsSearchQuery ? 'No transforms found' : 'No transforms available'} +

+
+ )} + +
+ + + {/* Flows Tab */} + + {/* Flows Search */} +
+
+ + setFlowsSearchQuery(e.target.value)} + className="h-8 pl-7 text-sm" + /> +
+
+ + {/* Flows List */} +
+ + {isLoadingFlows ? ( + // Skeleton loading state + Array.from({ length: 3 }).map((_, index) => ( + + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + )) + ) : filteredFlows.length > 0 ? ( + filteredFlows.map((flow: Flow) => ( + handleSelectTransform(flow)} + > + +
+
+ + {flow.name} +
+ + {flow.description && ( + + {flow.description || "No description available"} + + )} + +
+
Created {formatDistanceToNow(flow.created_at, { addSuffix: true })}
+
Updated {formatDistanceToNow(flow.last_updated_at, { addSuffix: true })}
+
+
+ + + )) + ) : ( +
+

+ {flowsSearchQuery ? 'No flows found' : 'No flows available'} +

+
+ )} + +
+ + + - +
diff --git a/flowsint-app/src/renderer/src/components/layout/breadcrumb.tsx b/flowsint-app/src/renderer/src/components/layout/breadcrumb.tsx index fbae258..570f0d3 100644 --- a/flowsint-app/src/renderer/src/components/layout/breadcrumb.tsx +++ b/flowsint-app/src/renderer/src/components/layout/breadcrumb.tsx @@ -11,12 +11,12 @@ import { Link } from "@tanstack/react-router" import { useQuery } from "@tanstack/react-query" import { investigationService } from "@/api/investigation-service" import { sketchService } from "@/api/sketch-service" -import { transformService } from "@/api/transfrom-service" import { Skeleton } from "@/components/ui/skeleton" import { Home } from "lucide-react" +import { flowService } from "@/api/flow-service" export function PathBreadcrumb() { - const { investigationId, id, type, transformId } = useParams({ strict: false }) + const { investigationId, id, type, flowId } = useParams({ strict: false }) const location = useLocation() const { data: investigation, isLoading: isInvestigationLoading } = useQuery({ @@ -31,13 +31,13 @@ export function PathBreadcrumb() { enabled: !!id && type === "graph", }) - const { data: transform, isLoading: isTransformLoading } = useQuery({ - queryKey: ["transform", transformId], - queryFn: () => transformService.getById(transformId!), - enabled: !!transformId, + const { data: flow, isLoading: isFlowLoading } = useQuery({ + queryKey: ["flow", flowId], + queryFn: () => flowService.getById(flowId!), + enabled: !!flowId, }) - const isTransformsPage = location.pathname.includes('/transforms') + const isFlowPage = location.pathname.includes('/flows') return (
@@ -50,25 +50,25 @@ export function PathBreadcrumb() { - {isTransformsPage ? ( + {isFlowPage ? ( <> - - Transforms + + Flows - {transformId && ( + {flowId && ( <> - {isTransformLoading ? ( + {isFlowLoading ? ( ) : ( - transform?.name || "(Unnamed transform)" + flow?.name || "(Unnamed flow)" )} diff --git a/flowsint-app/src/renderer/src/components/layout/secondary-navigation.tsx b/flowsint-app/src/renderer/src/components/layout/secondary-navigation.tsx index 312e64b..cd20e9a 100644 --- a/flowsint-app/src/renderer/src/components/layout/secondary-navigation.tsx +++ b/flowsint-app/src/renderer/src/components/layout/secondary-navigation.tsx @@ -1,7 +1,7 @@ import { useLocation, useParams } from "@tanstack/react-router" import InvestigationList from "../investigations/investigation-list" import GraphNavigation from "../graphs/graph-navigation" -import TransformNavigation from "../transforms/transform-navigation" +import TransformNavigation from "../flows/flow-navigation" import SketchList from "../investigations/sketch-list" import AnalysesList from "../analyses/analyses-list" @@ -10,7 +10,7 @@ const SecondaryNavigation = () => { const { id, investigationId, type } = useParams({ strict: false }) const { pathname } = useLocation() - if (!investigationId && !id && !pathname.startsWith("/dashboard/transforms")) { + if (!investigationId && !id && !pathname.startsWith("/dashboard/flows")) { return (
@@ -41,7 +41,7 @@ const SecondaryNavigation = () => { } - if (pathname.startsWith("/dashboard/transforms")) { + if (pathname.startsWith("/dashboard/flows")) { return (
diff --git a/flowsint-app/src/renderer/src/components/layout/sidebar.tsx b/flowsint-app/src/renderer/src/components/layout/sidebar.tsx index 7b3c78b..337b793 100644 --- a/flowsint-app/src/renderer/src/components/layout/sidebar.tsx +++ b/flowsint-app/src/renderer/src/components/layout/sidebar.tsx @@ -27,7 +27,7 @@ export function Sidebar() { const navItems: NavItem[] = [ { icon: Home, label: "Dashboard", href: "/dashboard/" }, - { icon: Workflow, label: "Flows", href: "/dashboard/transforms" }, + { icon: Workflow, label: "Flows", href: "/dashboard/flows" }, { icon: Lock, label: "Vault", href: "/dashboard/vault" }, ] diff --git a/flowsint-app/src/renderer/src/hooks/use-launch-flow.ts b/flowsint-app/src/renderer/src/hooks/use-launch-flow.ts new file mode 100644 index 0000000..d8c5818 --- /dev/null +++ b/flowsint-app/src/renderer/src/hooks/use-launch-flow.ts @@ -0,0 +1,27 @@ +import { toast } from "sonner" +import { useConfirm } from "@/components/use-confirm-dialog" +import { flowService } from "@/api/flow-service" + +export function useLaunchFlow(askUser: boolean = false) { + const { confirm } = useConfirm() + const launchFlow = async (values: string[], flow_id: string, sketch_id: string | null | undefined) => { + if (!sketch_id) return toast.error("Could not find the graph.") + if (askUser) { + const confirmed = await confirm({ + title: `${flow_id} scan`, + message: `You're about to launch ${flow_id} flow on ${values.length} items.`, + }) + if (!confirmed) return + } + const body = JSON.stringify({ values, sketch_id }) + toast.promise(flowService.launch(flow_id, body), { + loading: "Loading...", + success: () => `Scan on "${values.join(",")}" has been launched.`, + error: () => `An error occurred launching flow.`, + }) + return + } + return { + launchFlow, + } +} diff --git a/flowsint-app/src/renderer/src/hooks/use-launch-transform.ts b/flowsint-app/src/renderer/src/hooks/use-launch-transform.ts index d2baaa8..08c845c 100644 --- a/flowsint-app/src/renderer/src/hooks/use-launch-transform.ts +++ b/flowsint-app/src/renderer/src/hooks/use-launch-transform.ts @@ -4,17 +4,17 @@ import { transformService } from "@/api/transfrom-service" export function useLaunchTransform(askUser: boolean = false) { const { confirm } = useConfirm() - const launchTransform = async (values: string[], transform_id: string, sketch_id: string | null | undefined) => { + const launchTransform = async (values: string[], transformName: string, sketch_id: string | null | undefined) => { if (!sketch_id) return toast.error("Could not find the graph.") if (askUser) { const confirmed = await confirm({ - title: `${transform_id} scan`, - message: `You're about to launch ${transform_id} transform on ${values.length} items.`, + title: `${transformName} scan`, + message: `You're about to launch ${transformName} transform on ${values.length} items.`, }) if (!confirmed) return } const body = JSON.stringify({ values, sketch_id }) - toast.promise(transformService.launch(transform_id, body), { + toast.promise(transformService.launch(transformName, body), { loading: "Loading...", success: () => `Scan on "${values.join(",")}" has been launched.`, error: () => `An error occurred launching transform.`, diff --git a/flowsint-app/src/renderer/src/routeTree.gen.ts b/flowsint-app/src/renderer/src/routeTree.gen.ts index 6533831..d2608ac 100644 --- a/flowsint-app/src/renderer/src/routeTree.gen.ts +++ b/flowsint-app/src/renderer/src/routeTree.gen.ts @@ -18,9 +18,9 @@ import { Route as AuthDashboardRouteImport } from './routes/_auth.dashboard' import { Route as AuthDashboardIndexRouteImport } from './routes/_auth.dashboard.index' import { Route as AuthDashboardVaultRouteImport } from './routes/_auth.dashboard.vault' import { Route as AuthDashboardToolsRouteImport } from './routes/_auth.dashboard.tools' -import { Route as AuthDashboardTransformsIndexRouteImport } from './routes/_auth.dashboard.transforms.index' import { Route as AuthDashboardInvestigationsIndexRouteImport } from './routes/_auth.dashboard.investigations.index' -import { Route as AuthDashboardTransformsTransformIdRouteImport } from './routes/_auth.dashboard.transforms.$transformId' +import { Route as AuthDashboardFlowsIndexRouteImport } from './routes/_auth.dashboard.flows.index' +import { Route as AuthDashboardFlowsFlowIdRouteImport } from './routes/_auth.dashboard.flows.$flowId' import { Route as AuthDashboardInvestigationsInvestigationIdIndexRouteImport } from './routes/_auth.dashboard.investigations.$investigationId.index' import { Route as AuthDashboardInvestigationsInvestigationIdTypeIdRouteImport } from './routes/_auth.dashboard.investigations.$investigationId.$type.$id' @@ -68,22 +68,21 @@ const AuthDashboardToolsRoute = AuthDashboardToolsRouteImport.update({ path: '/tools', getParentRoute: () => AuthDashboardRoute, } as any) -const AuthDashboardTransformsIndexRoute = - AuthDashboardTransformsIndexRouteImport.update({ - id: '/transforms/', - path: '/transforms/', - getParentRoute: () => AuthDashboardRoute, - } as any) const AuthDashboardInvestigationsIndexRoute = AuthDashboardInvestigationsIndexRouteImport.update({ id: '/investigations/', path: '/investigations/', getParentRoute: () => AuthDashboardRoute, } as any) -const AuthDashboardTransformsTransformIdRoute = - AuthDashboardTransformsTransformIdRouteImport.update({ - id: '/transforms/$transformId', - path: '/transforms/$transformId', +const AuthDashboardFlowsIndexRoute = AuthDashboardFlowsIndexRouteImport.update({ + id: '/flows/', + path: '/flows/', + getParentRoute: () => AuthDashboardRoute, +} as any) +const AuthDashboardFlowsFlowIdRoute = + AuthDashboardFlowsFlowIdRouteImport.update({ + id: '/flows/$flowId', + path: '/flows/$flowId', getParentRoute: () => AuthDashboardRoute, } as any) const AuthDashboardInvestigationsInvestigationIdIndexRoute = @@ -108,9 +107,9 @@ export interface FileRoutesByFullPath { '/dashboard/tools': typeof AuthDashboardToolsRoute '/dashboard/vault': typeof AuthDashboardVaultRoute '/dashboard/': typeof AuthDashboardIndexRoute - '/dashboard/transforms/$transformId': typeof AuthDashboardTransformsTransformIdRoute + '/dashboard/flows/$flowId': typeof AuthDashboardFlowsFlowIdRoute + '/dashboard/flows': typeof AuthDashboardFlowsIndexRoute '/dashboard/investigations': typeof AuthDashboardInvestigationsIndexRoute - '/dashboard/transforms': typeof AuthDashboardTransformsIndexRoute '/dashboard/investigations/$investigationId': typeof AuthDashboardInvestigationsInvestigationIdIndexRoute '/dashboard/investigations/$investigationId/$type/$id': typeof AuthDashboardInvestigationsInvestigationIdTypeIdRoute } @@ -122,9 +121,9 @@ export interface FileRoutesByTo { '/dashboard/tools': typeof AuthDashboardToolsRoute '/dashboard/vault': typeof AuthDashboardVaultRoute '/dashboard': typeof AuthDashboardIndexRoute - '/dashboard/transforms/$transformId': typeof AuthDashboardTransformsTransformIdRoute + '/dashboard/flows/$flowId': typeof AuthDashboardFlowsFlowIdRoute + '/dashboard/flows': typeof AuthDashboardFlowsIndexRoute '/dashboard/investigations': typeof AuthDashboardInvestigationsIndexRoute - '/dashboard/transforms': typeof AuthDashboardTransformsIndexRoute '/dashboard/investigations/$investigationId': typeof AuthDashboardInvestigationsInvestigationIdIndexRoute '/dashboard/investigations/$investigationId/$type/$id': typeof AuthDashboardInvestigationsInvestigationIdTypeIdRoute } @@ -139,9 +138,9 @@ export interface FileRoutesById { '/_auth/dashboard/tools': typeof AuthDashboardToolsRoute '/_auth/dashboard/vault': typeof AuthDashboardVaultRoute '/_auth/dashboard/': typeof AuthDashboardIndexRoute - '/_auth/dashboard/transforms/$transformId': typeof AuthDashboardTransformsTransformIdRoute + '/_auth/dashboard/flows/$flowId': typeof AuthDashboardFlowsFlowIdRoute + '/_auth/dashboard/flows/': typeof AuthDashboardFlowsIndexRoute '/_auth/dashboard/investigations/': typeof AuthDashboardInvestigationsIndexRoute - '/_auth/dashboard/transforms/': typeof AuthDashboardTransformsIndexRoute '/_auth/dashboard/investigations/$investigationId/': typeof AuthDashboardInvestigationsInvestigationIdIndexRoute '/_auth/dashboard/investigations/$investigationId/$type/$id': typeof AuthDashboardInvestigationsInvestigationIdTypeIdRoute } @@ -156,9 +155,9 @@ export interface FileRouteTypes { | '/dashboard/tools' | '/dashboard/vault' | '/dashboard/' - | '/dashboard/transforms/$transformId' + | '/dashboard/flows/$flowId' + | '/dashboard/flows' | '/dashboard/investigations' - | '/dashboard/transforms' | '/dashboard/investigations/$investigationId' | '/dashboard/investigations/$investigationId/$type/$id' fileRoutesByTo: FileRoutesByTo @@ -170,9 +169,9 @@ export interface FileRouteTypes { | '/dashboard/tools' | '/dashboard/vault' | '/dashboard' - | '/dashboard/transforms/$transformId' + | '/dashboard/flows/$flowId' + | '/dashboard/flows' | '/dashboard/investigations' - | '/dashboard/transforms' | '/dashboard/investigations/$investigationId' | '/dashboard/investigations/$investigationId/$type/$id' id: @@ -186,9 +185,9 @@ export interface FileRouteTypes { | '/_auth/dashboard/tools' | '/_auth/dashboard/vault' | '/_auth/dashboard/' - | '/_auth/dashboard/transforms/$transformId' + | '/_auth/dashboard/flows/$flowId' + | '/_auth/dashboard/flows/' | '/_auth/dashboard/investigations/' - | '/_auth/dashboard/transforms/' | '/_auth/dashboard/investigations/$investigationId/' | '/_auth/dashboard/investigations/$investigationId/$type/$id' fileRoutesById: FileRoutesById @@ -266,13 +265,6 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AuthDashboardToolsRouteImport parentRoute: typeof AuthDashboardRoute } - '/_auth/dashboard/transforms/': { - id: '/_auth/dashboard/transforms/' - path: '/transforms' - fullPath: '/dashboard/transforms' - preLoaderRoute: typeof AuthDashboardTransformsIndexRouteImport - parentRoute: typeof AuthDashboardRoute - } '/_auth/dashboard/investigations/': { id: '/_auth/dashboard/investigations/' path: '/investigations' @@ -280,11 +272,18 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AuthDashboardInvestigationsIndexRouteImport parentRoute: typeof AuthDashboardRoute } - '/_auth/dashboard/transforms/$transformId': { - id: '/_auth/dashboard/transforms/$transformId' - path: '/transforms/$transformId' - fullPath: '/dashboard/transforms/$transformId' - preLoaderRoute: typeof AuthDashboardTransformsTransformIdRouteImport + '/_auth/dashboard/flows/': { + id: '/_auth/dashboard/flows/' + path: '/flows' + fullPath: '/dashboard/flows' + preLoaderRoute: typeof AuthDashboardFlowsIndexRouteImport + parentRoute: typeof AuthDashboardRoute + } + '/_auth/dashboard/flows/$flowId': { + id: '/_auth/dashboard/flows/$flowId' + path: '/flows/$flowId' + fullPath: '/dashboard/flows/$flowId' + preLoaderRoute: typeof AuthDashboardFlowsFlowIdRouteImport parentRoute: typeof AuthDashboardRoute } '/_auth/dashboard/investigations/$investigationId/': { @@ -308,9 +307,9 @@ interface AuthDashboardRouteChildren { AuthDashboardToolsRoute: typeof AuthDashboardToolsRoute AuthDashboardVaultRoute: typeof AuthDashboardVaultRoute AuthDashboardIndexRoute: typeof AuthDashboardIndexRoute - AuthDashboardTransformsTransformIdRoute: typeof AuthDashboardTransformsTransformIdRoute + AuthDashboardFlowsFlowIdRoute: typeof AuthDashboardFlowsFlowIdRoute + AuthDashboardFlowsIndexRoute: typeof AuthDashboardFlowsIndexRoute AuthDashboardInvestigationsIndexRoute: typeof AuthDashboardInvestigationsIndexRoute - AuthDashboardTransformsIndexRoute: typeof AuthDashboardTransformsIndexRoute AuthDashboardInvestigationsInvestigationIdIndexRoute: typeof AuthDashboardInvestigationsInvestigationIdIndexRoute AuthDashboardInvestigationsInvestigationIdTypeIdRoute: typeof AuthDashboardInvestigationsInvestigationIdTypeIdRoute } @@ -319,10 +318,9 @@ const AuthDashboardRouteChildren: AuthDashboardRouteChildren = { AuthDashboardToolsRoute: AuthDashboardToolsRoute, AuthDashboardVaultRoute: AuthDashboardVaultRoute, AuthDashboardIndexRoute: AuthDashboardIndexRoute, - AuthDashboardTransformsTransformIdRoute: - AuthDashboardTransformsTransformIdRoute, + AuthDashboardFlowsFlowIdRoute: AuthDashboardFlowsFlowIdRoute, + AuthDashboardFlowsIndexRoute: AuthDashboardFlowsIndexRoute, AuthDashboardInvestigationsIndexRoute: AuthDashboardInvestigationsIndexRoute, - AuthDashboardTransformsIndexRoute: AuthDashboardTransformsIndexRoute, AuthDashboardInvestigationsInvestigationIdIndexRoute: AuthDashboardInvestigationsInvestigationIdIndexRoute, AuthDashboardInvestigationsInvestigationIdTypeIdRoute: diff --git a/flowsint-app/src/renderer/src/routes/_auth.dashboard.transforms.$transformId.tsx b/flowsint-app/src/renderer/src/routes/_auth.dashboard.flows.$flowId.tsx similarity index 51% rename from flowsint-app/src/renderer/src/routes/_auth.dashboard.transforms.$transformId.tsx rename to flowsint-app/src/renderer/src/routes/_auth.dashboard.flows.$flowId.tsx index a482ed5..5e35559 100644 --- a/flowsint-app/src/renderer/src/routes/_auth.dashboard.transforms.$transformId.tsx +++ b/flowsint-app/src/renderer/src/routes/_auth.dashboard.flows.$flowId.tsx @@ -1,20 +1,20 @@ import { createFileRoute } from '@tanstack/react-router' -import { transformService } from '@/api/transfrom-service' -import Editor from '@/components/transforms/editor' +import Editor from '@/components/flows/editor' import Loader from '@/components/loader' +import { flowService } from '@/api/flow-service' -export const Route = createFileRoute('/_auth/dashboard/transforms/$transformId')({ - loader: async ({ params: { transformId } }) => { +export const Route = createFileRoute('/_auth/dashboard/flows/$flowId')({ + loader: async ({ params: { flowId } }) => { return { - transform: await transformService.getById(transformId), + flow: await flowService.getById(flowId), } }, - component: TranformPage, + component: FlowPage, pendingComponent: () => (
-

Loading transform...

+

Loading flow...

), @@ -22,7 +22,7 @@ export const Route = createFileRoute('/_auth/dashboard/transforms/$transformId')

- Error loading transform + Error loading flow

{error.message}

@@ -30,14 +30,14 @@ export const Route = createFileRoute('/_auth/dashboard/transforms/$transformId') ), }) -function TranformPage() { - const { transform } = Route.useLoaderData() +function FlowPage() { + const { flow } = Route.useLoaderData() return ( ) } \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/routes/_auth.dashboard.transforms.index.tsx b/flowsint-app/src/renderer/src/routes/_auth.dashboard.flows.index.tsx similarity index 77% rename from flowsint-app/src/renderer/src/routes/_auth.dashboard.transforms.index.tsx rename to flowsint-app/src/renderer/src/routes/_auth.dashboard.flows.index.tsx index 85e0f47..92a5bdf 100644 --- a/flowsint-app/src/renderer/src/routes/_auth.dashboard.transforms.index.tsx +++ b/flowsint-app/src/renderer/src/routes/_auth.dashboard.flows.index.tsx @@ -1,5 +1,4 @@ import { createFileRoute } from '@tanstack/react-router' -import { transformService } from '@/api/transfrom-service' import { useQuery } from '@tanstack/react-query' import { Button } from '@/components/ui/button' import { PlusIcon, FileCode2, Clock, FileX } from 'lucide-react' @@ -9,33 +8,34 @@ import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@/com import { formatDistanceToNow } from 'date-fns' import { Badge } from '@/components/ui/badge' import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs' -import NewTransform from '@/components/transforms/new-transform' +import NewFlow from '@/components/flows/new-flow' +import { flowService } from '@/api/flow-service' -interface Transform { +interface Flow { id: string name: string description?: string category?: string[] created_at: string updated_at?: string - transform_schema?: any + flow_schema?: any } -export const Route = createFileRoute('/_auth/dashboard/transforms/')({ - component: TransformsPage, +export const Route = createFileRoute('/_auth/dashboard/flows/')({ + component: FlowPage, }) -function TransformsPage() { +function FlowPage() { const navigate = useNavigate() - const { data: transforms, isLoading } = useQuery({ - queryKey: ["transforms"], - queryFn: () => transformService.get(), + const { data: flows, isLoading } = useQuery({ + queryKey: ["flow"], + queryFn: () => flowService.get(), }) // Get all unique categories - const categories = transforms?.reduce((acc: string[], transform) => { - if (transform.category) { - transform.category.forEach(cat => { + const categories = flows?.reduce((acc: string[], flow) => { + if (flow.category) { + flow.category.forEach(cat => { if (!acc.includes(cat)) acc.push(cat) }) } @@ -53,16 +53,16 @@ function TransformsPage() {

Flows

- Create and manage your transform flows. + Create and manage your flow flows.

- + - +
@@ -72,21 +72,21 @@ function TransformsPage() {
- ) : !transforms?.length ? ( + ) : !flows?.length ? (

No flow yet

- Get started by creating your first flow. You can use transforms to process and manipulate your data in powerful ways. + Get started by creating your first flow. You can use flows to process and manipulate your data in powerful ways.

- + - +
) : ( @@ -105,39 +105,39 @@ function TransformsPage() { {allCategories.map((category) => (
- {transforms - ?.filter(transform => + {flows + ?.filter(flow => category === 'All' ? true : category === 'Uncategorized' - ? !transform.category?.length - : transform.category?.includes(category) + ? !flow.category?.length + : flow.category?.includes(category) ) - .map((transform) => ( + .map((flow) => ( navigate({ to: `/dashboard/transforms/${transform.id}` })} + onClick={() => navigate({ to: `/dashboard/flows/${flow.id}` })} >
- {transform.name || "(Unnamed transform)"} + {flow.name || "(Unnamed flow)"}
- {transform.description || "No description provided"} + {flow.description || "No description provided"}
- {formatDistanceToNow(new Date(transform.updated_at || transform.created_at), { addSuffix: true })} + {formatDistanceToNow(new Date(flow.updated_at || flow.created_at), { addSuffix: true })}
- {transform.category?.map((cat) => ( + {flow.category?.map((cat) => ( {cat} diff --git a/flowsint-app/src/renderer/src/routes/_auth.dashboard.investigations.$investigationId.index.tsx b/flowsint-app/src/renderer/src/routes/_auth.dashboard.investigations.$investigationId.index.tsx index 9d49daf..fafefb6 100644 --- a/flowsint-app/src/renderer/src/routes/_auth.dashboard.investigations.$investigationId.index.tsx +++ b/flowsint-app/src/renderer/src/routes/_auth.dashboard.investigations.$investigationId.index.tsx @@ -1,48 +1,49 @@ -import { createFileRoute } from '@tanstack/react-router' +import { createFileRoute, useNavigate } from '@tanstack/react-router' import { investigationService } from '@/api/investigation-service' -import { InvestigationSketches } from '@/components/dashboard/investigation-sketches' -import { InvestigationAnalyses } from '@/components/dashboard/investigation-analyses' +import { analysisService } from '@/api/analysis-service' +import { Button } from '@/components/ui/button' +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card' +import { Badge } from '@/components/ui/badge' +import { Separator } from '@/components/ui/separator' +import { Plus, Calendar, User, FileText, BarChart3, Clock, ArrowRight } from 'lucide-react' +import { useMutation, useQueryClient } from '@tanstack/react-query' +import { toast } from 'sonner' +import { formatDistanceToNow } from 'date-fns' function InvestigationSkeleton() { return (
-
- {/* Investigation Cards Skeleton */} +
+ {/* Header Skeleton */}
-
-
-
-
-
-
-
-
-
-
-
- {Array.from({ length: 8 }).map((_, i) => ( -
- ))} +
+
+
+
+
+
- {/* Analysis Cards Skeleton */} -
-
-
-
+ {/* Stats Cards Skeleton */} +
+ {Array.from({ length: 3 }).map((_, i) => ( +
+ ))} +
+ + {/* Content Skeleton */} +
+ {Array.from({ length: 2 }).map((_, i) => ( +
+
+ {Array.from({ length: 4 }).map((_, j) => ( +
+ ))} +
-
-
-
-
-
-
- {Array.from({ length: 8 }).map((_, i) => ( -
- ))} -
+ ))}
@@ -61,16 +62,314 @@ export const Route = createFileRoute('/_auth/dashboard/investigations/$investiga function InvestigationPage() { const { investigation } = Route.useLoaderData() + const navigate = useNavigate() + const queryClient = useQueryClient() + + const createAnalysisMutation = useMutation({ + mutationFn: async () => { + const newAnalysis = { + title: "Untitled Analysis", + investigation_id: investigation.id, + content: {}, + } + return analysisService.create(JSON.stringify(newAnalysis)) + }, + onSuccess: async (data) => { + queryClient.invalidateQueries({ queryKey: ["analyses", "investigation", investigation.id] }) + toast.success("New analysis created") + // Navigate to the new analysis page + navigate({ + to: "/dashboard/investigations/$investigationId/$type/$id", + params: { + investigationId: investigation.id, + type: "analysis", + id: data.id + } + }) + }, + onError: (error) => { + toast.error("Failed to create analysis: " + (error instanceof Error ? error.message : "Unknown error")) + } + }) + + const sketchCount = investigation.sketches?.length || 0 + const analysisCount = investigation.analyses?.length || 0 + const lastUpdated = formatDistanceToNow(new Date(investigation.last_updated_at), { addSuffix: true }) return (
- {/* Main Content */} -
- {/* Investigation Sketches */} - +
+ {/* Header Section */} +
+
+
+

{investigation.name}

+

+ {investigation.description || "No description provided"} +

+
+
+ +
+
+ +
+
+ + Created {formatDistanceToNow(new Date(investigation.created_at), { addSuffix: true })} +
+
+ + Updated {lastUpdated} +
+
+ + {investigation.owner ? `${investigation.owner.first_name || ''} ${investigation.owner.last_name || ''}`.trim() || 'Unknown' : "Unknown"} +
+ + {investigation.status} + +
+
- {/* Investigation Analyses */} - + {/* Stats Cards */} +
+ + + Total Sketches + + + +
{sketchCount}
+

+ Visual data representations +

+ + + + + + Total Analyses + + + +
{analysisCount}
+

+ Documented findings +

+ + + + + + Last Activity + + + +
{lastUpdated}
+

+ Since last update +

+ + +
+ + + + {/* Content Sections */} +
+ {/* Sketches Section */} +
+
+
+ +

Sketches

+ {sketchCount} +
+ +
+ + {sketchCount === 0 ? ( + + + +

No sketches yet

+

+ Create your first sketch to start visualizing your investigation data and building relationships between entities. +

+ + + + ) : ( +
+ {investigation.sketches?.slice(0, 8).map((sketch: any) => ( + navigate({ + to: "/dashboard/investigations/$investigationId/$type/$id", + params: { + investigationId: investigation.id, + type: "graph", + id: sketch.id + } + })} + > +
+ +
+
+ +
+ + {sketch.status || "active"} + +
+ +
+

+ {sketch.title} +

+

+ {sketch.description || "No description provided"} +

+
+ +
+
+ + + {formatDistanceToNow(new Date(sketch.last_updated_at), { addSuffix: true })} + +
+
+
+ + + ))} +
+ )} +
+ + {/* Analyses Section */} +
+
+
+ +

Analyses

+ {analysisCount} +
+ +
+ + {analysisCount === 0 ? ( + + + +

No analyses yet

+

+ Create your first analysis to start documenting your findings, observations, and investigative notes. +

+ + + + ) : ( +
+ {investigation.analyses?.slice(0, 8).map((analysis: any) => ( + navigate({ + to: "/dashboard/investigations/$investigationId/$type/$id", + params: { + investigationId: investigation.id, + type: "analysis", + id: analysis.id + } + })} + > +
+ +
+
+ +
+ + Analysis + +
+ +
+

+ {analysis.title} +

+

+ {analysis.description || "No description provided"} +

+
+ +
+
+ + + {formatDistanceToNow(new Date(analysis.last_updated_at), { addSuffix: true })} + +
+
+
+ + + ))} +
+ )} +
+
); diff --git a/flowsint-app/src/renderer/src/stores/transform-store.ts b/flowsint-app/src/renderer/src/stores/flow-store.ts similarity index 78% rename from flowsint-app/src/renderer/src/stores/transform-store.ts rename to flowsint-app/src/renderer/src/stores/flow-store.ts index ec650a5..d039086 100644 --- a/flowsint-app/src/renderer/src/stores/transform-store.ts +++ b/flowsint-app/src/renderer/src/stores/flow-store.ts @@ -16,33 +16,33 @@ import { type ScannerNodeData } from "@/types/transform" export type NodeData = ScannerNodeData -export type TransformNode = Node -export type TransformEdge = Edge +export type FlowNode = Node +export type FlowEdge = Edge export interface TransformState { // Node State - nodes: TransformNode[] - selectedNode: TransformNode | null + nodes: FlowNode[] + selectedNode: FlowNode | null // Edge State - edges: TransformEdge[] + edges: FlowEdge[] // UI State loading: boolean openParamsDialog: boolean - openTransformSheet: boolean + openFlowSheet: boolean // Node Actions - setNodes: (nodes: TransformNode[] | ((prev: TransformNode[]) => TransformNode[])) => void + setNodes: (nodes: FlowNode[] | ((prev: FlowNode[]) => FlowNode[])) => void onNodesChange: OnNodesChange - setSelectedNode: (node: TransformNode | null) => void + setSelectedNode: (node: FlowNode | null) => void deleteNode: (nodeId: string) => void - updateNode: (node: TransformNode) => void + updateNode: (node: FlowNode) => void // Edge Actions - setEdges: (edges: TransformEdge[] | ((prev: TransformEdge[]) => TransformEdge[])) => void + setEdges: (edges: FlowEdge[] | ((prev: FlowEdge[]) => FlowEdge[])) => void onEdgesChange: OnEdgesChange onConnect: OnConnect // UI Actions setLoading: (loading: boolean) => void - setOpenParamsDialog: (openParamsDialog: boolean, node?: TransformNode) => void - setOpenTransformSheet: (openTransformSheet: boolean, node?: TransformNode) => void + setOpenParamsDialog: (openParamsDialog: boolean, node?: FlowNode) => void + setOpenFlowSheet: (openFlowSheet: boolean, node?: FlowNode) => void } // ================================ @@ -61,26 +61,26 @@ const defaultMarkerEnd: EdgeMarker = { // TRANSFORM STORE IMPLEMENTATION // ================================ -export const useTransformStore = create((set, get) => ({ +export const useFlowStore = create((set, get) => ({ // ================================ // STATE INITIALIZATION // ================================ // Node State - nodes: [] as TransformNode[], + nodes: [] as FlowNode[], selectedNode: null, // Edge State - edges: [] as TransformEdge[], + edges: [] as FlowEdge[], // UI State loading: false, openParamsDialog: false, - openTransformSheet: false, + openFlowSheet: false, // ================================ // NODE ACTIONS // ================================ setNodes: (nodes) => set({ nodes: typeof nodes === 'function' ? nodes(get().nodes) : nodes }), onNodesChange: (changes) => { set({ - nodes: applyNodeChanges(changes, get().nodes) as TransformNode[], + nodes: applyNodeChanges(changes, get().nodes) as FlowNode[], }) }, setSelectedNode: (node) => set({ selectedNode: node }), @@ -111,7 +111,7 @@ export const useTransformStore = create((set, get) => ({ toast.error(`Cannot connect ${connection.sourceHandle} to ${connection.targetHandle}.`) return } - const edge: TransformEdge = { + const edge: FlowEdge = { id: `${connection.source}-${connection.target}`, source: connection.source!, target: connection.target!, @@ -139,15 +139,15 @@ export const useTransformStore = create((set, get) => ({ } set({ openParamsDialog }) }, - setOpenTransformSheet: (openTransformSheet, node) => { + setOpenFlowSheet: (openFlowSheet, node) => { // Only allow opening the dialog if there's a selected node if (node) { set({ selectedNode: node }) } - if (openTransformSheet && !get().selectedNode) { + if (openFlowSheet && !get().selectedNode) { toast.error("Please select a node first to add a connector.") return } - set({ openTransformSheet }) + set({ openFlowSheet }) } })) diff --git a/flowsint-app/src/renderer/src/stores/graph-settings-store.ts b/flowsint-app/src/renderer/src/stores/graph-settings-store.ts index 8a8f64e..ef9cda4 100644 --- a/flowsint-app/src/renderer/src/stores/graph-settings-store.ts +++ b/flowsint-app/src/renderer/src/stores/graph-settings-store.ts @@ -25,7 +25,7 @@ const DEFAULT_SETTINGS = { description: "Distance between different graph depths when using DAG (directed acyclic graph) layout mode" }, linkDirectionalArrowRelPos: { - value: 0.99, + value: 1, min: 0, max: 1, step: 0.01, @@ -170,7 +170,7 @@ export const useGraphSettingsStore = create()( return flatSettings }, resetSettings: () => set({ settings: DEFAULT_SETTINGS, currentPreset: null }), - + // Force Presets Implementation getPresets: () => ({ 'Tight Clusters': { @@ -210,12 +210,12 @@ export const useGraphSettingsStore = create()( collisionRadius: 12 } }), - + applyPreset: (presetName: string) => { const presets = get().getPresets(); const preset = presets[presetName]; if (!preset) return; - + set((state) => { const newSettings = { ...state.settings }; Object.entries(preset).forEach(([key, value]) => { @@ -226,18 +226,18 @@ export const useGraphSettingsStore = create()( }; } }); - return { + return { settings: newSettings, currentPreset: presetName }; }); } - } ), + }), { name: 'graph-controls-storage', - partialize: (state) => ({ + partialize: (state) => ({ settings: state.settings, - currentPreset: state.currentPreset + currentPreset: state.currentPreset }), } )); diff --git a/flowsint-app/src/renderer/src/stores/node-display-settings.ts b/flowsint-app/src/renderer/src/stores/node-display-settings.ts index 4e4c92b..f73c884 100644 --- a/flowsint-app/src/renderer/src/stores/node-display-settings.ts +++ b/flowsint-app/src/renderer/src/stores/node-display-settings.ts @@ -113,60 +113,61 @@ export const ITEM_TYPES: ItemType[] = [ ] const DEFAULT_COLORS: Record = { - individual: "#A9CCF4", // medium blue - phone: "#A0D7CF", // soft teal - location: "#EAAFAF", // dusty rose - email: "#C4B9ED", // lavender - ip: "#F0C19E", // pale orange - socialprofile: "#D4A5D4", // dusty violet - organization: "#D1C0AF", // taupe - vehicle: "#E9CD89", // wheat - car: "#D4C4A8", // warm beige - motorcycle: "#C8B5A3", // warm taupe - boat: "#B8D4E3", // soft blue - plane: "#E2D1C3", // light taupe - website: "#E7B8D2", // dusty rose - domain: "#A6D0BF", // sage - subdomain: "#9DCBE4", // sky blue - document: "#C1C6CD", // cool gray - financial: "#F4B8A8", // coral - event: "#A2D4BF", // mint - device: "#EAC597", // peach - media: "#E4B1AD", // terracotta - education: "#ABC0DA", // steel blue - relationship: "#D9C2BC", // dusty mauve - online_activity: "#B5D1A9", // sage - digital_footprint: "#D9B0B0", // brick - username: "#C7BEE4", // periwinkle - credential: "#F7D154", // medium gray - biometric: "#AEB3B9", // slate gray - siret: "#ADB9C6", // blue-gray - siren: "#9FAAB8", // dark slate - cryptowallet: "#F7D154", // gold - cryptotransaction: "#E6D4A0", // warm yellow - cryptonft: "#D4E6A0", // soft lime - asn: "#EAAFAF", // warm peach - cidr: "#B8E6B8", // soft mint - whois: "#D4B5D4", // dusty lavender - gravatar: "#A8D8E8", // pale blue - breach: "#E6B8B8", // soft rose - webtracker: "#F0E6A0", // warm yellow - session: "#D4E6A0", // soft lime - dns: "#B8E6B8", // soft mint - ssl: "#E6D4A0", // warm yellow - message: "#C4B9ED", // lavender - malware: "#1DA7A8", // soft rose - weapon: "#F4B8A8", // coral - script: "#D4A5D4", // dusty violet - reputation: "#A9CCF4", // medium blue - risk: "#EAAFAF", // dusty rose - file: "#C1C6CD", // cool gray - bank: "#F7D154", // gold - creditcard: "#0253A4", // warm yellow - alias: "#D4A5D4", // dusty violet - affiliation: "#A6D0BF", // sage - phrase: "#D4A5D4" // warm beige -} + individual: "#E07A7A", // stronger pastel blue + phone: "#5FC9B5", // teal-green + location: "#E57373", // warm rose + email: "#8E7CC3", // lavender purple + ip: "#F4A261", // orange + socialprofile: "#B569B9", // purple pink + organization: "#BCA18A", // taupe + vehicle: "#E1B84D", // golden wheat + car: "#BFAF7A", // olive beige + motorcycle: "#A78B6C", // warm taupe + boat: "#6FB1C5", // blue teal + plane: "#C1A78E", // light brown + website: "#D279A6", // dusty rose + domain: "#66A892", // sage green + subdomain: "#5AA1C8", // sky blue + document: "#8F9CA3", // cool gray + financial: "#E98973", // coral + event: "#6DBBA2", // mint green + device: "#E3A857", // peach orange + media: "#C97C73", // terracotta + education: "#6C8CBF", // steel blue + relationship: "#B18B84", // mauve brown + online_activity: "#7EAD6F", // sage green + digital_footprint: "#B97777", // brick + username: "#8B83C1", // periwinkle purple + credential: "#D4B030", // gold + biometric: "#7F868D", // slate gray + siret: "#7D8B99", // blue-gray + siren: "#687684", // dark slate + cryptowallet: "#D4B030", // gold yellow + cryptotransaction: "#BFA750", // warm yellow + cryptonft: "#A5BF50", // lime green + asn: "#D97474", // warm peach + cidr: "#80BF80", // mint green + whois: "#9B6F9B", // lavender violet + gravatar: "#6CB7CA", // pale cyan + breach: "#CC7A7A", // warm rose + webtracker: "#C7BF50", // warm yellow + session: "#A8BF50", // lime green + dns: "#80BF9F", // mint teal + ssl: "#BFAF80", // warm sand + message: "#897FC9", // lavender + malware: "#4AA29E", // teal + weapon: "#E98973", // coral brown + script: "#A36FA3", // dusty violet + reputation: "#6FA8DC", // steel blue + risk: "#D97474", // dusty rose + file: "#8F9CA3", // cool gray + bank: "#D4B030", // gold + creditcard: "#285E8E", // deep blue + alias: "#A36FA3", // violet + affiliation: "#66A892", // sage + phrase: "#BFA77A" // warm beige +}; + // Définition des icônes par défaut pour chaque type d'élément const DEFAULT_ICONS: Record = { diff --git a/flowsint-app/src/renderer/src/types/flow.ts b/flowsint-app/src/renderer/src/types/flow.ts new file mode 100644 index 0000000..e9c3f1b --- /dev/null +++ b/flowsint-app/src/renderer/src/types/flow.ts @@ -0,0 +1,36 @@ +// ================================ +// FLOW TYPE DEFINITIONS +// ================================ + +export interface Flow { + id: string; + class_name: string; + name: string; + module: string; + description: string; + documentation: string; + category: string; + created_at: string; + last_updated_at: string; +} + +// ================================ +// FLOW DATA STRUCTURES +// ================================ + +export interface FlowsData { + [category: string]: Flow[]; +} + +export interface FlowData { + items: FlowsData; +} + +// ================================ +// COMPONENT PROPS INTERFACES +// ================================ + +export interface FlowItemProps { + flow: Flow; + category: string; +} diff --git a/flowsint-app/src/renderer/src/types/index.ts b/flowsint-app/src/renderer/src/types/index.ts index 23d99f7..3c5e747 100644 --- a/flowsint-app/src/renderer/src/types/index.ts +++ b/flowsint-app/src/renderer/src/types/index.ts @@ -2,6 +2,7 @@ export * from "./analysis"; export * from "./chat"; export * from "./common"; export * from "./event"; +export * from "./flow"; export * from "./graph"; export * from "./investigation"; export * from "./profile"; diff --git a/flowsint-app/src/renderer/src/types/investigation.ts b/flowsint-app/src/renderer/src/types/investigation.ts index 8170907..07c5c83 100644 --- a/flowsint-app/src/renderer/src/types/investigation.ts +++ b/flowsint-app/src/renderer/src/types/investigation.ts @@ -1,11 +1,13 @@ import { type Sketch } from "./sketch" import { type Profile } from "./profile" +import { type Analysis } from "./analysis" export interface Investigation { id: string name: string description: string sketches: Sketch[] + analyses: Analysis[] created_at: string last_updated_at: string owner: Profile diff --git a/flowsint-app/src/renderer/src/types/transform.ts b/flowsint-app/src/renderer/src/types/transform.ts index 5f1e371..5c63d63 100644 --- a/flowsint-app/src/renderer/src/types/transform.ts +++ b/flowsint-app/src/renderer/src/types/transform.ts @@ -80,30 +80,36 @@ export interface ScannerNodeProps { } // ================================ -// ADDITIONAL TRANSFORM TYPES +// TRANSFORM TYPE DEFINITIONS // ================================ -export type ScannerTree = { - id: string; - name: string, - items: ScannerTree[] - // Add other item properties -}; - export interface Transform { id: string; + class_name: string; name: string; - description?: string; - nodes: NodeData[]; - edges: EdgeData[]; - created_at?: string; - updated_at?: string; - owner?: Profile; + module: string; + description: string; + documentation: string; + category: string; } -export type NodesData = { - items: ScannerTree[]; - initialEdges?: Edge[]; - initialNodes?: Node[]; - transform?: Transform; -}; \ No newline at end of file +// ================================ +// TRANSFORM DATA STRUCTURES +// ================================ + +export interface TransformsData { + [category: string]: Transform[]; +} + +export interface TransformData { + items: TransformsData; +} + +// ================================ +// COMPONENT PROPS INTERFACES +// ================================ + +export interface TransformItemProps { + transform: Transform; + category: string; +} \ No newline at end of file diff --git a/flowsint-core/src/flowsint_core/core/celery.py b/flowsint-core/src/flowsint_core/core/celery.py index 1da8010..5a61b36 100644 --- a/flowsint-core/src/flowsint_core/core/celery.py +++ b/flowsint-core/src/flowsint_core/core/celery.py @@ -8,6 +8,7 @@ celery = Celery( include=[ "flowsint_core.tasks.event", "flowsint_core.tasks.transform", + "flowsint_core.tasks.flow", ], ) diff --git a/flowsint-core/src/flowsint_core/core/models.py b/flowsint-core/src/flowsint_core/core/models.py index f12a8a1..c097353 100644 --- a/flowsint-core/src/flowsint_core/core/models.py +++ b/flowsint-core/src/flowsint_core/core/models.py @@ -50,6 +50,7 @@ class Investigation(Base): sketches = relationship("Sketch", back_populates="investigation") analyses = relationship("Analysis", back_populates="investigation") chats = relationship("Chat", back_populates="investigation") + owner = relationship("Profile", foreign_keys=[owner_id]) __table_args__ = ( Index("idx_investigations_id", "id"), Index("idx_investigations_owner_id", "owner_id"), @@ -203,7 +204,7 @@ class Transform(Base): name = mapped_column(Text, nullable=False) description = mapped_column(Text, nullable=True) category = mapped_column(ARRAY(Text), nullable=True) - transform_schema = mapped_column(JSON, nullable=True) + flow_schema = mapped_column(JSON, nullable=True) created_at = mapped_column(DateTime(timezone=True), server_default=func.now()) last_updated_at = mapped_column(DateTime(timezone=True), server_default=func.now()) diff --git a/flowsint-core/src/flowsint_core/core/orchestrator.py b/flowsint-core/src/flowsint_core/core/orchestrator.py index ba8f8bc..60550cc 100644 --- a/flowsint-core/src/flowsint_core/core/orchestrator.py +++ b/flowsint-core/src/flowsint_core/core/orchestrator.py @@ -3,7 +3,7 @@ from datetime import datetime import time from pydantic import ValidationError from .scanner_base import Scanner -from .registry import ScannerRegistry +from .registry import TransformRegistry from .types import FlowBranch, FlowStep from .logger import Logger from ..utils import to_json_serializable @@ -229,18 +229,18 @@ class TransformOrchestrator(Scanner): for node in scanner_nodes: node_id = node.nodeId - # Extract scanner name from nodeId (assuming format like "scanner_name-1234567890") - scanner_name = node_id.split("-")[0] + # Extract scanner name from nodeId (assuming format like "transform_name-1234567890") + transform_name = node_id.split("-")[0] - if not ScannerRegistry.scanner_exists(scanner_name): - raise ValueError(f"Scanner '{scanner_name}' not found in registry") + if not TransformRegistry.transform_exists(transform_name): + raise ValueError(f"Scanner '{transform_name}' not found in registry") # Pass the step params to the scanner instance scanner_params = ( node.params if hasattr(node, "params") and node.params else {} ) - scanner = ScannerRegistry.get_scanner( - scanner_name, + scanner = TransformRegistry.get_scanner( + transform_name, self.sketch_id, self.scan_id, neo4j_conn=self.neo4j_conn, @@ -383,12 +383,12 @@ class TransformOrchestrator(Scanner): ) continue - scanner_name = scanner.name() + transform_name = scanner.name() step_start_time = time.time() step_result = { "nodeId": node_id, - "scanner": scanner_name, + "scanner": transform_name, "status": "error", # Default to error, will update on success } @@ -398,7 +398,7 @@ class TransformOrchestrator(Scanner): "branch_id": branch_id, "branch_name": branch_name, "node_id": node_id, - "scanner_name": scanner_name, + "transform_name": transform_name, "inputs": to_json_serializable(scanner_inputs), "outputs": None, "status": "running", @@ -432,7 +432,7 @@ class TransformOrchestrator(Scanner): outputs = await scanner.execute(scanner_inputs) if not isinstance(outputs, (dict, list)): raise ValueError( - f"Scanner '{scanner_name}' returned unsupported output format" + f"Scanner '{transform_name}' returned unsupported output format" ) # Cache the results scanner_results_cache[cache_key] = outputs diff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.py index eaa053b..c27a57e 100644 --- a/flowsint-core/src/flowsint_core/core/registry.py +++ b/flowsint-core/src/flowsint_core/core/registry.py @@ -26,9 +26,10 @@ from flowsint_transforms.individuals.to_org import IndividualToOrgScanner from flowsint_transforms.organizations.to_infos import OrgToInfosScanner from flowsint_transforms.websites.to_webtrackers import WebsiteToWebtrackersScanner from flowsint_transforms.n8n.connector import N8nConnector +from flowsint_transforms.domains.to_root_domain import DomainToRootDomain -class ScannerRegistry: +class TransformRegistry: _scanners: Dict[str, Type[Scanner]] = {} @@ -37,7 +38,7 @@ class ScannerRegistry: cls._scanners[scanner_class.name()] = scanner_class @classmethod - def scanner_exists(cls, name: str) -> bool: + def transform_exists(cls, name: str) -> bool: return name in cls._scanners @classmethod @@ -99,26 +100,27 @@ class ScannerRegistry: # Register all scanners -ScannerRegistry.register(ReverseResolveScanner) -ScannerRegistry.register(ResolveScanner) -ScannerRegistry.register(SubdomainScanner) -ScannerRegistry.register(WhoisScanner) -ScannerRegistry.register(GeolocationScanner) -ScannerRegistry.register(MaigretScanner) -ScannerRegistry.register(IpToAsnScanner) -ScannerRegistry.register(AsnToCidrsScanner) -ScannerRegistry.register(CidrToIpsScanner) -ScannerRegistry.register(OrgToAsnScanner) -ScannerRegistry.register(DomainToAsnScanner) -ScannerRegistry.register(CryptoWalletAddressToTransactions) -ScannerRegistry.register(CryptoWalletAddressToNFTs) -ScannerRegistry.register(DomainToWebsiteScanner) -ScannerRegistry.register(WebsiteToCrawler) -ScannerRegistry.register(WebsiteToLinks) -ScannerRegistry.register(WebsiteToDomainScanner) -ScannerRegistry.register(EmailToGravatarScanner) -ScannerRegistry.register(EmailToBreachesScanner) -ScannerRegistry.register(IndividualToOrgScanner) -ScannerRegistry.register(OrgToInfosScanner) -ScannerRegistry.register(WebsiteToWebtrackersScanner) -ScannerRegistry.register(N8nConnector) +TransformRegistry.register(ReverseResolveScanner) +TransformRegistry.register(ResolveScanner) +TransformRegistry.register(SubdomainScanner) +TransformRegistry.register(WhoisScanner) +TransformRegistry.register(GeolocationScanner) +TransformRegistry.register(MaigretScanner) +TransformRegistry.register(IpToAsnScanner) +TransformRegistry.register(AsnToCidrsScanner) +TransformRegistry.register(CidrToIpsScanner) +TransformRegistry.register(OrgToAsnScanner) +TransformRegistry.register(DomainToAsnScanner) +TransformRegistry.register(CryptoWalletAddressToTransactions) +TransformRegistry.register(CryptoWalletAddressToNFTs) +TransformRegistry.register(DomainToWebsiteScanner) +TransformRegistry.register(WebsiteToCrawler) +TransformRegistry.register(WebsiteToLinks) +TransformRegistry.register(WebsiteToDomainScanner) +TransformRegistry.register(EmailToGravatarScanner) +TransformRegistry.register(EmailToBreachesScanner) +TransformRegistry.register(IndividualToOrgScanner) +TransformRegistry.register(OrgToInfosScanner) +TransformRegistry.register(WebsiteToWebtrackersScanner) +TransformRegistry.register(DomainToRootDomain) +TransformRegistry.register(N8nConnector) diff --git a/flowsint-core/src/flowsint_core/tasks/flow.py b/flowsint-core/src/flowsint_core/tasks/flow.py new file mode 100644 index 0000000..a81b405 --- /dev/null +++ b/flowsint-core/src/flowsint_core/tasks/flow.py @@ -0,0 +1,97 @@ +import os +import uuid +from dotenv import load_dotenv +from typing import List, Optional +from celery import states +from ..core.celery import celery +from ..core.orchestrator import TransformOrchestrator +from ..core.postgre_db import SessionLocal, get_db +from ..core.graph_db import Neo4jConnection +from ..core.vault import Vault +from ..core.types import FlowBranch +from ..core.models import Scan +from sqlalchemy.orm import Session +from ..core.logger import Logger +from ..core.enums import EventLevel +from flowsint_core.utils import to_json_serializable + +load_dotenv() + +URI = os.getenv("NEO4J_URI_BOLT") +URI = "bolt://localhost:7687" +USERNAME = os.getenv("NEO4J_USERNAME") +PASSWORD = os.getenv("NEO4J_PASSWORD") + +neo4j_connection = Neo4jConnection(URI, USERNAME, PASSWORD) +db: Session = next(get_db()) +logger = Logger() + + +@celery.task(name="run_flow", bind=True) +def run_flow( + self, + transform_branches, + values: List[str], + sketch_id: str | None, + owner_id: Optional[str] = None, +): + session = SessionLocal() + + try: + if not transform_branches: + raise ValueError("transform_branches not provided in the input transform") + + scan_id = uuid.UUID(self.request.id) + + scan = Scan( + id=scan_id, + status=EventLevel.PENDING, + sketch_id=uuid.UUID(sketch_id) if sketch_id else None, + ) + session.add(scan) + session.commit() + + # Create vault instance if owner_id is provided + vault = None + if owner_id: + try: + vault = Vault(session, uuid.UUID(owner_id)) + except Exception as e: + Logger.error( + sketch_id, {"message": f"Failed to create vault: {str(e)}"} + ) + + transform_branches = [FlowBranch(**branch) for branch in transform_branches] + scanner = TransformOrchestrator( + sketch_id=sketch_id, + scan_id=str(scan_id), + transform_branches=transform_branches, + neo4j_conn=neo4j_connection, + vault=vault, + ) + + # Use the synchronous scan method which internally handles the async operations + results = scanner.scan(values=values) + + scan.status = EventLevel.COMPLETED + scan.results = to_json_serializable(results) + session.commit() + + return {"result": scan.results} + + except Exception as ex: + session.rollback() + error_logs = f"An error occurred: {str(ex)}" + print(f"Error in task: {error_logs}") + + scan = session.query(Scan).filter(Scan.id == uuid.UUID(self.request.id)).first() + if scan: + scan.status = EventLevel.FAILED + scan.results = {"error": error_logs} + session.commit() + + self.update_state(state=states.FAILURE) + raise ex + + finally: + session.close() diff --git a/flowsint-core/src/flowsint_core/tasks/transform.py b/flowsint-core/src/flowsint_core/tasks/transform.py index 600f1fd..ade1241 100644 --- a/flowsint-core/src/flowsint_core/tasks/transform.py +++ b/flowsint-core/src/flowsint_core/tasks/transform.py @@ -1,14 +1,14 @@ import os import uuid +import asyncio from dotenv import load_dotenv from typing import List, Optional from celery import states +from flowsint_core.core.registry import TransformRegistry from ..core.celery import celery -from ..core.orchestrator import TransformOrchestrator from ..core.postgre_db import SessionLocal, get_db from ..core.graph_db import Neo4jConnection from ..core.vault import Vault -from ..core.types import FlowBranch from ..core.models import Scan from sqlalchemy.orm import Session from ..core.logger import Logger @@ -30,7 +30,7 @@ logger = Logger() @celery.task(name="run_transform", bind=True) def run_transform( self, - transform_branches, + transform_name: str, values: List[str], sketch_id: str | None, owner_id: Optional[str] = None, @@ -38,8 +38,6 @@ def run_transform( session = SessionLocal() try: - if not transform_branches: - raise ValueError("transform_branches not provided in the input transform") scan_id = uuid.UUID(self.request.id) @@ -61,17 +59,18 @@ def run_transform( sketch_id, {"message": f"Failed to create vault: {str(e)}"} ) - transform_branches = [FlowBranch(**branch) for branch in transform_branches] - scanner = TransformOrchestrator( + if not TransformRegistry.transform_exists(transform_name): + raise ValueError(f"Scanner '{transform_name}' not found in registry") + + scanner = TransformRegistry.get_scanner( + name=transform_name, sketch_id=sketch_id, - scan_id=str(scan_id), - transform_branches=transform_branches, + scan_id=scan_id, neo4j_conn=neo4j_connection, vault=vault, ) - # Use the synchronous scan method which internally handles the async operations - results = scanner.scan(values=values) + results = asyncio.run(scanner.execute(values=values)) scan.status = EventLevel.COMPLETED scan.results = to_json_serializable(results) diff --git a/flowsint-core/src/flowsint_core/tests/orchestrator.py b/flowsint-core/src/flowsint_core/tests/orchestrator.py index 97deb6e..7d0809a 100644 --- a/flowsint-core/src/flowsint_core/tests/orchestrator.py +++ b/flowsint-core/src/flowsint_core/tests/orchestrator.py @@ -7,7 +7,7 @@ import pytest def test_preprocess_valid_domains(): scanner = TransformOrchestrator( - "123", scanner_names=["domain_resolve_scanner", "domain_whois_scanner"] + "123", transform_names=["domain_resolve_scanner", "domain_whois_scanner"] ) assert isinstance(scanner.scanners, list) assert len(scanner.scanners) == 2 @@ -18,13 +18,13 @@ def test_preprocess_valid_domains(): ValueError, match="Scanner 'this_scan_is_wrong' not found in registry" ): TransformOrchestrator( - "123", scanner_names=["domain_resolve_scanner", "this_scan_is_wrong"] + "123", transform_names=["domain_resolve_scanner", "this_scan_is_wrong"] ) def test_execute_domain_subdomains_scanner(): domains = ["example.com"] - scanner = TransformOrchestrator("123", scanner_names=["domain_subdomains_scanner"]) + scanner = TransformOrchestrator("123", transform_names=["domain_subdomains_scanner"]) results = scanner.execute(values=domains) assert results["initial_values"] == domains assert results["scanners"] == ["domain_subdomains_scanner"] @@ -34,7 +34,7 @@ def test_execute_domain_subdomains_scanner(): def test_execute_domain_whois_scanner(): domains = ["example.com"] - scanner = TransformOrchestrator("123", scanner_names=["domain_whois_scanner"]) + scanner = TransformOrchestrator("123", transform_names=["domain_whois_scanner"]) results = scanner.execute(values=domains) assert results["initial_values"] == domains assert results["scanners"] == ["domain_whois_scanner"] @@ -44,7 +44,7 @@ def test_execute_domain_whois_scanner(): def test_execute_ip_resolve(): ips = ["91.199.212.73"] - scanner = TransformOrchestrator("123", scanner_names=["ip_reverse_resolve_scanner"]) + scanner = TransformOrchestrator("123", transform_names=["ip_reverse_resolve_scanner"]) results = scanner.execute(values=ips) assert results["initial_values"] == ips assert results["scanners"] == ["ip_reverse_resolve_scanner"] @@ -56,7 +56,7 @@ def test_execute_ip_resolve(): def test_execute_ip_resolve_and_whois(): ips = ["91.199.212.73"] scanner = TransformOrchestrator( - "123", scanner_names=["ip_reverse_resolve_scanner", "domain_whois_scanner"] + "123", transform_names=["ip_reverse_resolve_scanner", "domain_whois_scanner"] ) results = scanner.execute(values=ips) assert results["initial_values"] == ips @@ -66,7 +66,7 @@ def test_execute_ip_resolve_and_whois(): def test_execute_ip_resolve_and_whois_multiple(): ips = ["91.199.212.73", "76.76.21.21"] scanner = TransformOrchestrator( - "123", scanner_names=["ip_reverse_resolve_scanner", "domain_whois_scanner"] + "123", transform_names=["ip_reverse_resolve_scanner", "domain_whois_scanner"] ) results = scanner.execute(values=ips) assert results["initial_values"] == ips @@ -78,7 +78,7 @@ def test_execute_ip_resolve_and_whois_multiple(): ips = ["162.19.81.222"] scanner = TransformOrchestrator( "123", - scanner_names=[ + transform_names=[ "ip_reverse_resolve_scanner", "domain_whois_scanner", "domain_subdomains_scanner", @@ -96,7 +96,7 @@ def test_execute_ip_resolve_and_whois_multiple(): def test_execute_domain_whois_and_subdomains(): domains = ["alliage.io"] scanner = TransformOrchestrator( - "123", scanner_names=["domain_whois_scanner", "domain_subdomains_scanner"] + "123", transform_names=["domain_whois_scanner", "domain_subdomains_scanner"] ) results = scanner.execute(values=domains) assert results["initial_values"] == domains @@ -105,7 +105,7 @@ def test_execute_domain_whois_and_subdomains(): def test_execute_domain_subdomains(): domains = ["alliage.io"] - scanner = TransformOrchestrator("123", scanner_names=["domain_subdomains_scanner"]) + scanner = TransformOrchestrator("123", transform_names=["domain_subdomains_scanner"]) results = scanner.execute(values=domains) assert results["initial_values"] == domains assert results["scanners"] == ["domain_subdomains_scanner"] diff --git a/flowsint-core/src/flowsint_core/tests/test_registry.py b/flowsint-core/src/flowsint_core/tests/test_registry.py index ef0c7a7..66f0d52 100644 --- a/flowsint-core/src/flowsint_core/tests/test_registry.py +++ b/flowsint-core/src/flowsint_core/tests/test_registry.py @@ -1,5 +1,5 @@ import pytest -from flowsint_core.core.registry import ScannerRegistry +from flowsint_core.core.registry import TransformRegistry from flowsint_core.core.scanner_base import Scanner from flowsint_types.domain import Domain from flowsint_types.ip import Ip @@ -44,19 +44,19 @@ class MockScanner(Scanner): return [] -class TestScannerRegistry: - """Test suite for ScannerRegistry functionality""" +class TestTransformRegistry: + """Test suite for TransformRegistry functionality""" def setup_method(self): """Clear registry before each test""" - ScannerRegistry.clear() + TransformRegistry.clear() def test_register_scanner(self): """Test registering a scanner""" - ScannerRegistry.register(MockScanner) - assert ScannerRegistry.scanner_exists("mock_scanner") + TransformRegistry.register(MockScanner) + assert TransformRegistry.transform_exists("mock_scanner") - scanners = ScannerRegistry.list() + scanners = TransformRegistry.list() assert "mock_scanner" in scanners assert scanners["mock_scanner"]["class_name"] == "MockScanner" @@ -67,13 +67,13 @@ class TestScannerRegistry: pass with pytest.raises(ValueError, match="must inherit from Scanner"): - ScannerRegistry.register(NotAScanner) + TransformRegistry.register(NotAScanner) def test_get_scanner_instance(self): """Test getting a scanner instance""" - ScannerRegistry.register(MockScanner) + TransformRegistry.register(MockScanner) - scanner = ScannerRegistry.get_scanner( + scanner = TransformRegistry.get_scanner( "mock_scanner", sketch_id="test_sketch", scan_id="test_scan" ) @@ -84,37 +84,37 @@ class TestScannerRegistry: def test_get_nonexistent_scanner(self): """Test that getting a non-existent scanner raises exception""" with pytest.raises(Exception, match="Scanner 'nonexistent' not found"): - ScannerRegistry.get_scanner( + TransformRegistry.get_scanner( "nonexistent", sketch_id="test_sketch", scan_id="test_scan" ) def test_list_by_categories(self): """Test listing scanners by category""" - ScannerRegistry.register(MockScanner) + TransformRegistry.register(MockScanner) - by_category = ScannerRegistry.list_by_categories() + by_category = TransformRegistry.list_by_categories() assert "Test" in by_category assert len(by_category["Test"]) == 1 assert by_category["Test"][0]["name"] == "mock_scanner" def test_list_by_input_type(self): """Test listing scanners by input type""" - ScannerRegistry.register(MockScanner) + TransformRegistry.register(MockScanner) - domain_scanners = ScannerRegistry.list_by_input_type("Domain") + domain_scanners = TransformRegistry.list_by_input_type("Domain") assert len(domain_scanners) == 1 assert domain_scanners[0]["name"] == "mock_scanner" # Test with "any" input type - any_scanners = ScannerRegistry.list_by_input_type("any") + any_scanners = TransformRegistry.list_by_input_type("any") assert len(any_scanners) == 1 assert any_scanners[0]["name"] == "mock_scanner" def test_register_module(self): """Test registering a module path""" - ScannerRegistry.register_module("test.module.path") - assert "test.module.path" in ScannerRegistry._scanner_modules + TransformRegistry.register_module("test.module.path") + assert "test.module.path" in TransformRegistry._scanner_modules # Registering the same module again should not duplicate - ScannerRegistry.register_module("test.module.path") - assert ScannerRegistry._scanner_modules.count("test.module.path") == 1 + TransformRegistry.register_module("test.module.path") + assert TransformRegistry._scanner_modules.count("test.module.path") == 1 diff --git a/flowsint-core/src/flowsint_core/utils.py b/flowsint-core/src/flowsint_core/utils.py index 238eed7..5a6441a 100644 --- a/flowsint-core/src/flowsint_core/utils.py +++ b/flowsint-core/src/flowsint_core/utils.py @@ -159,7 +159,7 @@ def resolve_type(details: dict, schema_context: dict = None) -> str: return "any" -def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]: +def extract_input_schema_flow(model: Type[BaseModel]) -> Dict[str, Any]: adapter = TypeAdapter(model) schema = adapter.json_schema() @@ -225,7 +225,7 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]: if scanner_node and scanner_node["data"]["type"] == "scanner": scanners.append( { - "scanner_name": scanner_node["data"]["name"], + "transform_name": scanner_node["data"]["name"], "module": scanner_node["data"]["module"], "input": source_handle, "output": target_handle, @@ -238,7 +238,7 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]: "outputs": input_output, }, "scanners": scanners, - "scanner_names": [scanner["scanner_name"] for scanner in scanners], + "transform_names": [scanner["transform_name"] for scanner in scanners], } diff --git a/flowsint-transforms/src/flowsint_transforms/domains/to_root_domain.py b/flowsint-transforms/src/flowsint_transforms/domains/to_root_domain.py new file mode 100644 index 0000000..2e66da4 --- /dev/null +++ b/flowsint-transforms/src/flowsint_transforms/domains/to_root_domain.py @@ -0,0 +1,102 @@ +from typing import List, Union +from flowsint_transforms.utils import is_valid_domain, get_root_domain +from flowsint_core.core.scanner_base import Scanner +from flowsint_types.domain import Domain +from flowsint_core.core.logger import Logger + + +class DomainToRootDomain(Scanner): + """Subdomain to root domain.""" + + InputType = List[Domain] + OutputType = List[Domain] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Store mapping between original domains and their root domains + self.domain_root_mapping: List[tuple[Domain, Domain]] = [] + + @classmethod + def name(cls) -> str: + return "to_root_domain" + + @classmethod + def category(cls) -> str: + return "Domain" + + @classmethod + def key(cls) -> str: + return "domain" + + def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType: + cleaned: InputType = [] + for item in data: + domain_obj = None + if isinstance(item, str): + if is_valid_domain(item): + domain_obj = Domain(domain=item) + elif isinstance(item, dict) and "domain" in item: + if is_valid_domain(item["domain"]): + domain_obj = Domain(domain=item["domain"]) + elif isinstance(item, Domain): + domain_obj = item + if domain_obj: + cleaned.append(domain_obj) + return cleaned + + async def scan(self, data: InputType) -> OutputType: + results: OutputType = [] + self.domain_root_mapping = [] # Reset mapping + + for domain in data: + try: + root_domain_name = get_root_domain(domain.domain) + # Only add if it's different from the original domain + if root_domain_name != domain.domain: + root_domain = Domain(domain=root_domain_name, root=True) + results.append(root_domain) + # Store the mapping for postprocess + self.domain_root_mapping.append((domain, root_domain)) + + except Exception as e: + Logger.error( + self.sketch_id, + {"message": f"Error getting root domain for {domain.domain}: {e}"}, + ) + continue + + return results + + def postprocess(self, results: OutputType, original_input: InputType) -> OutputType: + # Use the mapping we created during scan to create relationships + for original_domain, root_domain in self.domain_root_mapping: + if not self.neo4j_conn: + continue + + # Create root domain node + self.create_node("domain", "domain", root_domain.domain, type="domain") + + # Create original domain node + self.create_node("domain", "domain", original_domain.domain, type="domain") + + # Create relationship from root domain to original domain + self.create_relationship( + "domain", + "domain", + root_domain.domain, + "domain", + "domain", + original_domain.domain, + "HAS_SUBDOMAIN", + ) + + self.log_graph_message( + f"{root_domain.domain} -> HAS_SUBDOMAIN -> {original_domain.domain}" + ) + + return results + + +# Make types available at module level for easy access +InputType = DomainToRootDomain.InputType +OutputType = DomainToRootDomain.OutputType diff --git a/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py b/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py index 130f7a6..9b40b1f 100644 --- a/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py +++ b/flowsint-transforms/src/flowsint_transforms/ips/cidr_to_ips.py @@ -16,6 +16,10 @@ class CidrToIpsScanner(Scanner): @classmethod def name(cls) -> str: return "cidr_to_ips_scanner" + + @classmethod + def key(cls) -> str: + return "network" @classmethod def category(cls) -> str: diff --git a/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py b/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py index daa8838..1f9542b 100644 --- a/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py +++ b/flowsint-transforms/src/flowsint_transforms/ips/reverse_resolve.py @@ -86,8 +86,34 @@ class ReverseResolveScanner(Scanner): return results def postprocess( - self, results: OutputType, input_data: InputType = None + self, results: OutputType, original_input: InputType ) -> OutputType: + # Create nodes and relationships for each resolved domain + for ip_obj in original_input: + # Create IP node + self.create_node("ip", "address", ip_obj.address, type="ip") + + # Create domain nodes and relationships for each resolved domain + for domain_obj in results: + self.create_node( + "domain", + "domain", + domain_obj.domain, + type="domain" if "." not in domain_obj.domain.split(".")[1:] else "subdomain", + ) + self.create_relationship( + "ip", + "address", + ip_obj.address, + "domain", + "domain", + domain_obj.domain, + "REVERSE_RESOLVES_TO", + ) + self.log_graph_message( + f"Domain found for IP {ip_obj.address} -> {domain_obj.domain}" + ) + return results diff --git a/flowsint-transforms/utils.py b/flowsint-transforms/src/flowsint_transforms/utils.py similarity index 84% rename from flowsint-transforms/utils.py rename to flowsint-transforms/src/flowsint_transforms/utils.py index 238eed7..6a75267 100644 --- a/flowsint-transforms/utils.py +++ b/flowsint-transforms/src/flowsint_transforms/utils.py @@ -102,6 +102,59 @@ def is_root_domain(domain: str) -> bool: return False +def get_root_domain(domain: str) -> str: + """ + Extract the root domain from a given domain string. + + Args: + domain: The domain string (can be a subdomain or root domain) + + Returns: + The root domain (e.g., "example.com" from "sub.example.com" or "www.sub.example.com") + """ + try: + # Remove protocol if present + if "://" in domain: + parsed = urlparse(domain) + domain = parsed.hostname or domain + + # Split by dots + parts = domain.split(".") + + # Handle common country code TLDs that have 2 parts (e.g., .co.uk, .com.au, .org.uk) + common_cc_tlds = [ + ".co.uk", + ".com.au", + ".org.uk", + ".net.uk", + ".gov.uk", + ".ac.uk", + ".co.nz", + ".com.sg", + ".co.jp", + ".co.kr", + ".com.br", + ".com.mx", + ] + + # Check if the domain ends with a common country code TLD + for cc_tld in common_cc_tlds: + if domain.endswith(cc_tld): + # For country code TLDs, take the last 3 parts (e.g., example.co.uk) + if len(parts) >= 3: + return ".".join(parts[-3:]) + return domain + + # For regular TLDs, take the last 2 parts (e.g., example.com) + if len(parts) >= 2: + return ".".join(parts[-2:]) + + return domain + except Exception: + # If we can't parse it, return the original domain + return domain + + def is_valid_number(phone: str, region: str = "FR") -> None: """ Validates a phone number. Raises InvalidPhoneNumberError if invalid. @@ -159,7 +212,7 @@ def resolve_type(details: dict, schema_context: dict = None) -> str: return "any" -def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]: +def extract_input_schema_flow(model: Type[BaseModel]) -> Dict[str, Any]: adapter = TypeAdapter(model) schema = adapter.json_schema() @@ -225,7 +278,7 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]: if scanner_node and scanner_node["data"]["type"] == "scanner": scanners.append( { - "scanner_name": scanner_node["data"]["name"], + "transform_name": scanner_node["data"]["name"], "module": scanner_node["data"]["module"], "input": source_handle, "output": target_handle, @@ -238,7 +291,7 @@ def extract_transform(transform: Dict[str, Any]) -> Dict[str, Any]: "outputs": input_output, }, "scanners": scanners, - "scanner_names": [scanner["scanner_name"] for scanner in scanners], + "transform_names": [scanner["transform_name"] for scanner in scanners], } diff --git a/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py b/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py index ba4e8dc..5e5a98f 100644 --- a/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py +++ b/flowsint-transforms/src/flowsint_transforms/websites/to_webtrackers.py @@ -55,18 +55,19 @@ class WebsiteToWebtrackersScanner(Scanner): async def scan(self, data: InputType) -> OutputType: results: OutputType = [] - extractor = TrackingCodeExtractor() for website in data: try: # Extract tracking codes from the website - tracking_data = extractor.extract(str(website.url)) + extractor = TrackingCodeExtractor(str(website.url)) + extractor.fetch() + extractor.extract_codes() + tracking_codes = extractor.get_results() - for tracker_info in tracking_data: + for tracker_info in tracking_codes: tracker = WebTracker( - name=tracker_info.get("name", ""), - tracker_id=tracker_info.get("id", ""), - category=tracker_info.get("category", ""), + name=tracker_info.source, + tracker_id=tracker_info.code, website_url=str(website.url), ) results.append(tracker) @@ -83,9 +84,46 @@ class WebsiteToWebtrackersScanner(Scanner): return results - def postprocess( - self, results: OutputType, input_data: InputType = None - ) -> OutputType: + def postprocess(self, results: OutputType, original_input: InputType) -> OutputType: + # Create Neo4j relationships between websites and their corresponding trackers + if self.neo4j_conn: + # Group trackers by website using the mapping we created during scan + website_trackers = {} + for tracker, website in self.tracker_website_mapping: + website_url = str(website.url) + if website_url not in website_trackers: + website_trackers[website_url] = [] + website_trackers[website_url].append(tracker) + + # Create nodes and relationships for each website and its trackers + for website_url, trackers in website_trackers.items(): + # Create website node + self.create_node( + "website", "url", website_url, caption=website_url, type="website" + ) + + # Create tracker nodes and relationships + for tracker in trackers: + self.create_node( + "tracker", + "tracker_id", + tracker.tracker_id, + caption=tracker.name, + type="tracker" + ) + self.create_relationship( + "website", + "url", + website_url, + "tracker", + "tracker_id", + tracker.tracker_id, + "HAS_TRACKER", + ) + self.log_graph_message( + f"Found tracker {tracker.name} ({tracker.tracker_id}) for website {website_url}" + ) + return results diff --git a/flowsint-transforms/tests/scanners/emails/to_leaks.py b/flowsint-transforms/tests/scanners/emails/to_leaks.py index 0880682..77d68e3 100644 --- a/flowsint-transforms/tests/scanners/emails/to_leaks.py +++ b/flowsint-transforms/tests/scanners/emails/to_leaks.py @@ -7,7 +7,7 @@ from flowsint_types.breach import Breach scanner = EmailToBreachesScanner("sketch_123", "scan_123") -def test_scanner_name(): +def test_transform_name(): assert EmailToBreachesScanner.name() == "to_leaks"