feat: use of params for scanners

This commit is contained in:
dextmorgn
2025-07-20 22:39:11 +02:00
parent 41165a67f5
commit c2c7e7abb7
75 changed files with 2374 additions and 781 deletions
+3 -1
View File
@@ -171,4 +171,6 @@ cython_debug/
.ruff_cache/
# PyPI configuration file
.pypirc
.pypirc
transform_logs
@@ -0,0 +1,58 @@
"""drop_third_party_keys_create_keys_table
Revision ID: 9a3b9a199aa8
Revises: 0ab8ee0a782c
Create Date: 2025-07-20 12:13:27.871890
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = '9a3b9a199aa8'
down_revision: Union[str, None] = '0ab8ee0a782c'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
"""Upgrade schema."""
# Drop the old third_party_keys table if it exists
op.drop_table('third_party_keys')
# Create the new keys table
op.create_table('keys',
sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
sa.Column('name', sa.String(), nullable=False),
sa.Column('owner_id', postgresql.UUID(as_uuid=True), nullable=True),
sa.Column('encrypted_key', sa.String(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True),
sa.ForeignKeyConstraint(['owner_id'], ['profiles.id'], onupdate='CASCADE', ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id')
)
op.create_index('idx_keys_owner_id', 'keys', ['owner_id'], unique=False)
op.create_index('idx_keys_service', 'keys', ['name'], unique=False)
def downgrade() -> None:
"""Downgrade schema."""
# Drop the keys table
op.drop_index('idx_keys_service', table_name='keys')
op.drop_index('idx_keys_owner_id', table_name='keys')
op.drop_table('keys')
# Recreate the third_party_keys table
op.create_table('third_party_keys',
sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
sa.Column('service', sa.String(), nullable=False),
sa.Column('owner_id', postgresql.UUID(as_uuid=True), nullable=True),
sa.Column('encrypted_key', sa.String(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True),
sa.ForeignKeyConstraint(['owner_id'], ['profiles.id'], onupdate='CASCADE', ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id')
)
op.create_index('idx_keys_owner_id', 'third_party_keys', ['owner_id'], unique=False)
op.create_index('idx_keys_service', 'third_party_keys', ['service'], unique=False)
+20 -6
View File
@@ -39,12 +39,26 @@ def get_logs_by_sketch(
logs = query.limit(limit).all()
results = [Event(
id=str(log.id),
sketch_id=str(log.sketch_id) if log.sketch_id else None,
type=log.type,
payload=log.content
) for log in logs]
results = []
for log in logs:
# Ensure payload is always a dictionary
if isinstance(log.content, dict):
payload = log.content
elif isinstance(log.content, str):
payload = {"message": log.content}
elif log.content is None:
payload = {}
else:
# Handle other types by converting to string and wrapping
payload = {"content": str(log.content)}
results.append(Event(
id=str(log.id),
sketch_id=str(log.sketch_id) if log.sketch_id else None,
type=log.type,
payload=payload
))
return results
+35 -32
View File
@@ -2,53 +2,56 @@ from uuid import UUID, uuid4
from fastapi import APIRouter, HTTPException, Depends, status
from typing import List
from sqlalchemy.orm import Session
from pydantic import BaseModel
from app.core.postgre_db import get_db
from app.models.models import Profile, ThirdPartyKey
from app.models.models import Profile, Key
from app.api.deps import get_current_user
from app.api.schemas.key import ThirdPartyKeyRead, ThirdPartyKeyCreate
from app.api.schemas.key import KeyRead, KeyCreate
from datetime import datetime
router = APIRouter()
class ServiceInfo(BaseModel):
service: str
variable: str
url: str
active: bool
# Get the list of all services that require a key
@router.get("/services", response_model=List[ServiceInfo])
def get_services(db: Session = Depends(get_db)):
services = [
ServiceInfo(service="Mistral AI", variable="MISTRAL_API_KEY", url="https://mistral.ai/", active=True),
ServiceInfo(service="Etherscan", variable="ETHERSCAN_API_KEY", url="https://etherscan.io/", active=True),
ServiceInfo(service="HaveIBeenPwned", variable="HIBP_API_KEY", url="https://haveibeenpwned.com/", active=True),
ServiceInfo(service="Onyphe", variable="ONYPHE_API_KEY", url="https://www.onyphe.io/", active=False),
ServiceInfo(service="Shodan", variable="SHODAN_API_KEY", url="https://www.shodan.io/", active=False)
]
return services
def obfuscate_key(key: str) -> str:
"""Obfuscate a key by showing only the last 4 characters, replacing others with asterisks."""
if len(key) <= 4:
return key
return "*" * (len(key) - 4) + key[-4:]
# Get the list of all keys for a user
@router.get("", response_model=List[ThirdPartyKeyRead])
@router.get("", response_model=List[KeyRead])
def get_keys(db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
keys = db.query(ThirdPartyKey).filter(ThirdPartyKey.owner_id == current_user.id).all()
return keys
keys = db.query(Key).filter(Key.owner_id == current_user.id).all()
response_data = [KeyRead(
id=key.id,
owner_id=key.owner_id,
encrypted_key=obfuscate_key(key.encrypted_key),
name=key.name,
created_at=key.created_at
) for key in keys]
return response_data
# Get a key by ID
@router.get("/{id}", response_model=ThirdPartyKeyRead)
@router.get("/{id}", response_model=KeyRead)
def get_key_by_id(id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
key = db.query(ThirdPartyKey).filter(ThirdPartyKey.id == id, ThirdPartyKey.owner_id == current_user.id).first()
key = db.query(Key).filter(Key.id == id, Key.owner_id == current_user.id).first()
if not key:
raise HTTPException(status_code=404, detail="Key not found")
return key
# Create a response with obfuscated key
response_data = KeyRead(
id=key.id,
owner_id=key.owner_id,
encrypted_key=obfuscate_key(key.encrypted_key),
name=key.name,
created_at=key.created_at
)
return response_data
# Create a new key
@router.post("/create", response_model=ThirdPartyKeyRead, status_code=status.HTTP_201_CREATED)
def create_key(payload: ThirdPartyKeyCreate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
new_key = ThirdPartyKey(
@router.post("/create", response_model=KeyRead, status_code=status.HTTP_201_CREATED)
def create_key(payload: KeyCreate, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
new_key = Key(
id=uuid4(),
service=payload.service,
name=payload.name,
owner_id=current_user.id,
encrypted_key=payload.key,
created_at=datetime.utcnow(),
@@ -61,9 +64,9 @@ def create_key(payload: ThirdPartyKeyCreate, db: Session = Depends(get_db), curr
# Delete a key by ID
@router.delete("/{id}", status_code=status.HTTP_204_NO_CONTENT)
def delete_key(id: UUID, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
key = db.query(ThirdPartyKey).filter(ThirdPartyKey.id == id, ThirdPartyKey.owner_id == current_user.id).first()
key = db.query(Key).filter(Key.id == id, Key.owner_id == current_user.id).first()
if not key:
raise HTTPException(status_code=404, detail="Key not found")
db.delete(key)
db.commit()
return None
return None
+27 -3
View File
@@ -75,8 +75,28 @@ def delete_sketch(id: UUID, db: Session = Depends(get_db), current_user: Profile
db.commit()
@router.get("/{id}/graph")
async def get_sketch_nodes(id: str, db: Session = Depends(get_db), current_user: Profile = Depends(get_current_user)):
sketch = db.query(Sketch).filter(Sketch.id == id, Sketch.owner_id == current_user.id).first()
async def get_sketch_nodes(
id: str,
format: str = None,
db: Session = Depends(get_db),
# current_user: Profile = Depends(get_current_user)
):
"""
Get the nodes and relationships for a sketch.
Args:
id: The ID of the sketch
format: Optional format parameter. If "inline", returns inline relationships
db: The database session
current_user: The current user
Returns:
A dictionary containing the nodes and relationships for the sketch
nds: []
rls: []
Or if format=inline: List of inline relationship strings
"""
sketch = db.query(Sketch).filter(Sketch.id == id,
# Sketch.owner_id == current_user.id
).first()
if not sketch:
raise HTTPException(status_code=404, detail="Graph not found")
import random
@@ -123,11 +143,15 @@ async def get_sketch_nodes(id: str, db: Session = Depends(get_db), current_user:
"target": str(record["target"]),
"data": record["data"],
"caption": record["type"],
# "label": record["type"].lower(),
"label": record["type"].lower(),
}
for record in rels_result
]
if format == "inline":
from app.utils import get_inline_relationships
return get_inline_relationships(nodes, rels)
return {"nds": nodes, "rls": rels}
class NodeData(BaseModel):
+11 -4
View File
@@ -75,7 +75,10 @@ async def get_material_list():
"doc": scanner["doc"],
"inputs": scanner["inputs"],
"outputs": scanner["outputs"],
"type": "scanner"
"type": "scanner",
"params": scanner["params"],
"params_schema": scanner["params_schema"],
"requires_key": scanner["requires_key"]
}
for scanner in scanner_list
]
@@ -180,7 +183,7 @@ async def launch_transform(
edges
)
serializable_branches = [branch.dict() for branch in transform_branches]
task = celery.send_task("run_transform", args=[serializable_branches, payload.values, payload.sketch_id])
task = celery.send_task("run_transform", args=[serializable_branches, payload.values, payload.sketch_id, str(current_user.id)])
return {"id": task.id}
except Exception as e:
@@ -283,9 +286,10 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lis
key=lambda e: calculate_path_length(e.target)
)
def create_step(node_id: str, branch_id: str, depth: int, input_data: Dict[str, Any], is_input_node: bool, outputs: Dict[str, Any]) -> FlowStep:
def create_step(node_id: str, branch_id: str, depth: int, input_data: Dict[str, Any], is_input_node: bool, outputs: Dict[str, Any], node_params: Optional[Dict[str, Any]] = None) -> FlowStep:
return FlowStep(
nodeId=node_id,
params=node_params,
inputs={} if is_input_node else input_data,
outputs=outputs,
type="type" if is_input_node else "scanner",
@@ -320,8 +324,11 @@ def compute_transform_branches(initial_value: Any, nodes: List[Node], edges: Lis
# Store the outputs for future use
scanner_outputs[current_node_id] = current_outputs
# Extract node parameters
node_params = current_node.data.get("params", {})
# Create and add current step
current_step = create_step(current_node_id, branch_id, depth, input_data, is_input_node, current_outputs)
current_step = create_step(current_node_id, branch_id, depth, input_data, is_input_node, current_outputs, node_params)
steps.append(current_step)
path.append(current_node_id)
branch_visited.add(current_node_id)
+4 -4
View File
@@ -2,13 +2,13 @@ from .base import ORMBase
from pydantic import UUID4, BaseModel
from datetime import datetime
class ThirdPartyKeyCreate(BaseModel):
class KeyCreate(BaseModel):
key: str
service: str
name: str
class ThirdPartyKeyRead(ORMBase):
class KeyRead(ORMBase):
id: UUID4
owner_id: UUID4
encrypted_key: str
service: str
name: str
created_at: datetime
+27
View File
@@ -0,0 +1,27 @@
from typing import Protocol, Optional
import uuid
from sqlalchemy.orm import Session
from sqlalchemy import select
from app.models.models import Key
class VaultProtocol(Protocol):
def get_secret(self, vault_ref: str) -> Optional[str]:
...
class Vault(VaultProtocol):
def __init__(self, db: Session, owner_id: uuid.UUID):
if not owner_id:
raise ValueError("owner_id is required to use the vault.")
self.db = db
self.owner_id = owner_id
def get_secret(self, vault_ref: str) -> Optional[str]:
try:
ref_uuid = uuid.UUID(vault_ref)
stmt = select(Key).where(Key.id == ref_uuid)
except ValueError:
stmt = select(Key).where(Key.name == vault_ref)
stmt = stmt.where(Key.owner_id == self.owner_id)
result = self.db.execute(stmt)
row = result.scalars().first()
return row.encrypted_key if row else None
+1 -1
View File
@@ -59,5 +59,5 @@ app.include_router(events.router, prefix="/api/events", tags=["events"])
app.include_router(analysis.router, prefix="/api/analyses", tags=["analyses"])
app.include_router(chat.router, prefix="/api/chats", tags=["chats"])
app.include_router(scan.router, prefix="/api/scans", tags=["scans"])
app.include_router(keys.router, prefix="/api/third_party_keys", tags=["third_party_keys"])
app.include_router(keys.router, prefix="/api/keys", tags=["keys"])
app.include_router(types.router, prefix="/api/types", tags=["types"])
+4 -4
View File
@@ -194,16 +194,16 @@ class ChatMessage(Base):
Index("idx_messages_chat_id", "chat_id"),
)
class ThirdPartyKey(Base):
__tablename__ = "third_party_keys"
class Key(Base):
__tablename__ = "keys"
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
service = mapped_column(String, nullable=False)
name = mapped_column(String, nullable=False)
owner_id = mapped_column(PGUUID(as_uuid=True), ForeignKey("profiles.id", onupdate="CASCADE", ondelete="CASCADE"), nullable=True)
encrypted_key = mapped_column(String, nullable=False)
created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
__table_args__ = (
Index("idx_keys_owner_id", "owner_id"),
Index("idx_keys_service", "service"),
Index("idx_keys_service", "name"),
)
+113 -9
View File
@@ -1,18 +1,100 @@
from abc import ABC, abstractmethod
from typing import List, Dict, Any, Optional
from pydantic import ValidationError, BaseModel, Field, create_model
from pydantic.config import ConfigDict
from app.core.graph_db import Neo4jConnection
from app.core.logger import Logger
from app.core.vault import VaultProtocol
class Scanner(ABC):
class InvalidScannerParams(Exception):
pass
def build_params_model(params_schema: list) -> BaseModel:
"""
Build a strict Pydantic model from a params_schema.
Unknown fields will raise a validation error.
"""
fields: Dict[str, Any] = {}
for param in params_schema:
name = param["name"]
typ = str # You can later enhance this to support int, bool, etc.
required = param.get("required", False)
default = ... if required else param.get("default")
fields[name] = (Optional[typ], Field(default=default, description=param.get("description", "")))
model = create_model(
"ParamsModel",
__config__=ConfigDict(extra="forbid"),
**fields
)
return model
class Scanner(ABC):
def __init__(
self,
sketch_id: Optional[str],
scan_id: str,
sketch_id: Optional[str] = None,
scan_id: Optional[str] = None,
neo4j_conn: Optional[Neo4jConnection] = None,
params_schema: Optional[List[Dict[str, Any]]] = None,
vault: Optional[VaultProtocol] = None,
params: Optional[Dict[str, Any]] = None
):
self.scan_id = scan_id
self.scan_id = scan_id or "default"
self.sketch_id = sketch_id or "system"
self.neo4j_conn = neo4j_conn
self.vault = vault
self.params_schema = params_schema or []
self.ParamsModel = build_params_model(self.params_schema)
self.params: Dict[str, Any] = params or {}
# Params is filled synchronously by the constructor. This params is generally constructed of
# ApiKey references, not the key directly. The idea is that the real key values are resolved after calling
# async_init(), right before the execution.
async def async_init(self):
self.ParamsModel = build_params_model(self.params_schema)
Logger.warn(self.sketch_id, {"message": f"Params raw: {str(self.params)}"})
# Resolve parameters (e.g. replace vaultSecret by real secrets)
if self.params:
resolved_params = self.resolve_params()
Logger.warn(self.sketch_id, {"message": f"Resolved params: {str(resolved_params)}"})
else:
resolved_params = {}
# Strict validation after resolution
try:
validated = self.ParamsModel(**resolved_params)
self.params = validated.model_dump()
except ValidationError as e:
raise InvalidScannerParams(f"Scanner '{self.name()}' received invalid parameters: {e}")
def resolve_params(self) -> Dict[str, Any]:
resolved = {}
Logger.warn(self.sketch_id, {"message": f"Params schema: {str(self.params_schema)}"})
Logger.warn(self.sketch_id, {"message": f"Params: {str(self.params)}"})
Logger.warn(self.sketch_id, {"message": f"Params schema length: {len(self.params_schema)}"})
i = 1
for param in self.params_schema:
Logger.warn(self.sketch_id, {"message": f"Param {i}: {str(param)}"})
i += 1
if param["type"] == "vaultSecret":
resolved[param["name"]] = self.vault.get_secret(self.params[param["name"]])
else:
if self.params[param["name"]]:
resolved[param["name"]] = self.params[param["name"]]
elif param["default"] is not None:
resolved[param["name"]] = param["default"]
else:
continue
return resolved
@classmethod
def requires_key(self) -> bool:
return False
@classmethod
@abstractmethod
@@ -26,36 +108,58 @@ class Scanner(ABC):
@classmethod
@abstractmethod
def input_schema(cls) -> Dict[str, str]:
def key(cls) -> str:
"""Primary key on which the scanner operates (e.g. domain, IP, etc.)"""
pass
@classmethod
@abstractmethod
def output_schema(cls) -> Dict[str, str]:
def input_schema(cls) -> Dict[str, Any]:
pass
@classmethod
def get_params_schema(cls) -> List[Dict[str, Any]]:
"""Can be overridden in subclasses to declare required parameters"""
return []
@classmethod
@abstractmethod
def output_schema(cls) -> Dict[str, Any]:
pass
@abstractmethod
def scan(self, values: List[str]) -> List[Dict[str, Any]]:
async def scan(self, values: List[str]) -> List[Dict[str, Any]]:
pass
def set_params(self, params: Dict[str, Any]) -> None:
self.params = params
def get_params(self) -> Dict[str, Any]:
return self.params
def preprocess(self, values: List[str]) -> List[str]:
return values
def postprocess(self, results: List[Dict[str, Any]], input_data: List[str] = None) -> List[Dict[str, Any]]:
return results
def execute(self, values: List[str]) -> List[Dict[str, Any]]:
async def execute(self, values: List[str]) -> List[Dict[str, Any]]:
if self.name() != "transform_orchestrator":
Logger.info(self.sketch_id, {"message": f"Scanner {self.name()} started."})
try:
await self.async_init()
preprocessed = self.preprocess(values)
results = self.scan(preprocessed)
results = await self.scan(preprocessed)
processed = self.postprocess(results, preprocessed)
if self.name() != "transform_orchestrator":
Logger.completed(self.sketch_id, {"message": f"Scanner {self.name()} finished."})
return processed
except Exception as e:
if self.name() != "transform_orchestrator":
Logger.error(self.sketch_id, {"message": f"Scanner {self.name()} errored: '{str(e)}'."})
return []
@@ -1,21 +1,60 @@
import os
import socket
from typing import List, Dict, Any, TypeAlias, Union
from typing import List, Dict, Any, Optional, TypeAlias, Union
from pydantic import TypeAdapter
import requests
from app.scanners.base import Scanner
from app.types.wallet import CryptoWallet, CryptoNFT
from app.utils import resolve_type
from app.core.logger import Logger
from app.core.graph_db import Neo4jConnection
InputType: TypeAlias = List[CryptoWallet]
OutputType: TypeAlias = List[CryptoNFT]
ETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")
ETHERSCAN_API_KEY = os.getenv("ETHERSCAN_API_KEY")
class WalletAddressToNFTs(Scanner):
class CryptoWalletAddressToNFTs(Scanner):
"""Resolve NFTs for a wallet address (ETH)."""
def __init__(
self,
sketch_id: Optional[str] = None,
scan_id: Optional[str] = None,
neo4j_conn: Optional[Neo4jConnection] = None,
vault=None,
params: Optional[Dict[str, Any]] = None
):
super().__init__(
sketch_id=sketch_id,
scan_id=scan_id,
neo4j_conn=neo4j_conn,
params_schema=self.get_params_schema(),
vault=vault,
params=params
)
@classmethod
def requires_key(cls) -> bool:
return True
@classmethod
def get_params_schema(cls) -> List[Dict[str, Any]]:
"""Declare required parameters for this scanner"""
return [
{
"name": "ETHERSCAN_API_KEY",
"type": "vaultSecret",
"description": "The Etherscan API key to use for the transaction lookup.",
"required": True
},
{
"name": "ETHERSCAN_API_URL",
"type": "url",
"description": "The Etherscan API URL to use for the transaction lookup.",
"required": False,
"default": ETHERSCAN_API_URL
}
]
@classmethod
def name(cls) -> str:
return "wallet_to_nfts"
@@ -70,15 +109,23 @@ class WalletAddressToNFTs(Scanner):
def scan(self, data: InputType) -> OutputType:
results: OutputType = []
results: OutputType = []
params = self.get_params()
Logger.warn(self.sketch_id, {"message": f"{str(params)}"})
api_key = params["ETHERSCAN_API_KEY"]
api_url = params["ETHERSCAN_API_URL"]
if not api_key:
Logger.error(self.sketch_id, {"message": "ETHERSCAN_API_KEY is required"})
raise ValueError("ETHERSCAN_API_KEY is required")
for d in data:
try:
nfts = self._get_nfts(d.address)
nfts = self._get_nfts(d.address, api_key, api_url)
results.append(nfts)
except Exception as e:
print(f"Error resolving nfts for {d.address}: {e}")
return results
def _get_nfts(self, address: str) -> List[CryptoNFT]:
def _get_nfts(self, address: str, api_key: str, api_url: str) -> List[CryptoNFT]:
nfts = []
"""Get nfts for a wallet address."""
params = {
@@ -90,9 +137,9 @@ class WalletAddressToNFTs(Scanner):
"page": 1,
"offset": 10000,
"sort": "asc",
"apikey": ETHERSCAN_API_KEY
"apikey": api_key
}
response = requests.get(ETHERSCAN_API_URL, params=params)
response = requests.get(api_url, params=params)
data = response.json()
results = data["result"]
for tx in results:
@@ -1,23 +1,63 @@
import os
from typing import List, Dict, Any, TypeAlias, Union
from typing import List, Dict, Any, Optional, TypeAlias, Union
from pydantic import TypeAdapter
import requests
import requests.exceptions
from datetime import datetime
from app.scanners.base import Scanner
from app.types.wallet import CryptoWallet, CryptoWalletTransaction
from app.utils import resolve_type
from app.core.logger import Logger
from app.core.graph_db import Neo4jConnection
InputType: TypeAlias = List[CryptoWallet]
OutputType: TypeAlias = List[CryptoWalletTransaction]
ETHERSCAN_API_URL = os.getenv("ETHERSCAN_API_URL")
ETHERSCAN_API_KEY = os.getenv("ETHERSCAN_API_KEY")
def wei_to_eth(wei_str):
return int(wei_str) / 10**18
class CryptoWalletAddressToTransactions(Scanner):
"""Resolve transactions for a wallet address (ETH)."""
def __init__(
self,
sketch_id: Optional[str] = None,
scan_id: Optional[str] = None,
neo4j_conn: Optional[Neo4jConnection] = None,
vault=None,
params: Optional[Dict[str, Any]] = None
):
super().__init__(
sketch_id=sketch_id,
scan_id=scan_id,
neo4j_conn=neo4j_conn,
params_schema=self.get_params_schema(),
vault=vault,
params=params
)
@classmethod
def requires_key(cls) -> bool:
return True
@classmethod
def get_params_schema(cls) -> List[Dict[str, Any]]:
"""Declare required parameters for this scanner"""
return [
{
"name": "ETHERSCAN_API_KEY",
"type": "vaultSecret",
"description": "The Etherscan API key to use for the transaction lookup.",
"required": True
},
{
"name": "ETHERSCAN_API_URL",
"type": "url",
"description": "The Etherscan API URL to use for the transaction lookup.",
"required": False,
"default": ETHERSCAN_API_URL
}
]
@classmethod
def name(cls) -> str:
@@ -71,17 +111,23 @@ class CryptoWalletAddressToTransactions(Scanner):
cleaned.append(wallet_obj)
return cleaned
def scan(self, data: InputType) -> OutputType:
async def scan(self, data: InputType) -> OutputType:
results: OutputType = []
params = self.get_params()
api_key = params["ETHERSCAN_API_KEY"]
api_url = params["ETHERSCAN_API_URL"]
if not api_key:
Logger.error(self.sketch_id, {"message": "ETHERSCAN_API_KEY is required"})
raise ValueError("ETHERSCAN_API_KEY is required")
for d in data:
try:
transactions = self._get_transactions(d.address)
transactions = await self._get_transactions(d.address, api_key, api_url)
results.append(transactions)
except Exception as e:
Logger.error(self.sketch_id, {"message": f"Error resolving transactions for {d.address}: {e}"})
return results
def _get_transactions(self, address: str) -> List[CryptoWalletTransaction]:
async def _get_transactions(self, address: str, api_key: str, api_url: str) -> List[CryptoWalletTransaction]:
transactions = []
"""Get transactions for a wallet address."""
params = {
@@ -93,19 +139,45 @@ class CryptoWalletAddressToTransactions(Scanner):
"page": 1,
"offset": 100,
"sort": "asc",
"apikey": ETHERSCAN_API_KEY
"apikey": api_key
}
response = requests.get(ETHERSCAN_API_URL, params=params)
data = response.json()
results = data["result"]
try:
response = requests.get(api_url, params=params)
# Raise an exception for HTTP errors (4xx or 5xx status codes)
response.raise_for_status()
except requests.exceptions.ConnectionError as e:
raise ValueError(f"An error occurred connecting to {api_url}: Connection failed - {str(e)}")
except requests.exceptions.Timeout as e:
raise ValueError(f"An error occurred fetching {api_url}: Request timeout - {str(e)}")
except requests.exceptions.RequestException as e:
raise ValueError(f"An error occurred fetching {api_url}: {str(e)}")
try:
data = response.json()
except requests.exceptions.JSONDecodeError as e:
raise ValueError(f"An error occurred fetching {api_url}: Invalid JSON response - {str(e)}")
# Check if the API returned an error
if data.get("status") != "1":
error_message = data.get("message", "Unknown API error")
raise ValueError(f"An error occurred fetching {api_url}: {error_message}")
results = data.get("result", [])
for tx in results:
if tx["to"] !=None:
target = CryptoWallet(address=tx["to"])
# Properly determine source and target based on transaction data
source_address = tx["from"]
if tx["to"] is not None:
target_address = tx["to"]
else:
target = CryptoWallet(address=tx["contractAddress"])
# Contract creation transaction
target_address = tx["contractAddress"] if tx["contractAddress"] else address
transactions.append(CryptoWalletTransaction(
source=CryptoWallet(address=address),
target=target,
source=CryptoWallet(address=source_address),
target=CryptoWallet(address=target_address),
hash=tx["hash"],
value=wei_to_eth(tx["value"]),
timestamp=tx["timeStamp"],
@@ -136,7 +208,11 @@ class CryptoWalletAddressToTransactions(Scanner):
SET source.sketch_id = $sketch_id,
source.label = $source_address,
source.caption = $source_address,
source.type = "cryptowallet"
source.type = "cryptowallet",
target.sketch_id = $sketch_id,
target.label = $target_address,
target.caption = $target_address,
target.type = "cryptowallet"
"""
self.neo4j_conn.query(wallets_query, {
"source_address": tx.source.address,
@@ -146,8 +222,8 @@ class CryptoWalletAddressToTransactions(Scanner):
# Create transaction as an edge between wallets
tx_query = """
MATCH (source:cryptowallet {cryptowallet: $source})
MATCH (target:cryptowallet {cryptowallet: $target})
MATCH (source:cryptowallet {wallet: $source})
MATCH (target:cryptowallet {wallet: $target})
MERGE (source)-[tx:TRANSACTION {hash: $hash}]->(target)
SET tx.value = $value,
tx.timestamp = $timestamp,
@@ -124,7 +124,7 @@ class DomainToWebsiteScanner(Scanner):
for i, redirect_url in enumerate(website.redirects):
next_url = website.redirects[i + 1] if i + 1 < len(website.redirects) else str(website.url)
redirect_payload = {
"message": f"Redirect: {redirect_url} -> {next_url}"
"message": f"Redirect: {str(redirect_url)} -> {str(next_url)}"
}
Logger.info(self.sketch_id, redirect_payload)
@@ -27,6 +27,10 @@ class ReverseResolveScanner(Scanner):
def category(cls) -> str:
return "Ip"
@classmethod
def key(cls) -> str:
return "address"
@classmethod
def input_schema(cls) -> Dict[str, Any]:
adapter = TypeAdapter(InputType)
+220 -22
View File
@@ -2,21 +2,176 @@ from typing import List, Dict, Any
from uuid import UUID
from datetime import datetime
import uuid
import time
from pydantic import BaseModel, ValidationError
from app.scanners.base import Scanner
from app.scanners.registry import ScannerRegistry
from app.types.transform import FlowBranch, FlowStep
from app.core.logger import Logger
from app.utils import to_json_serializable
import asyncio
import json
import os
class TransformOrchestrator(Scanner):
def __init__(self, sketch_id: str, scan_id: str, transform_branches: List[FlowBranch], neo4j_conn=None):
super().__init__(sketch_id, scan_id, neo4j_conn=neo4j_conn)
"""
Orchestrator for running a list of scanners.
"""
def __init__(self, sketch_id: str, scan_id: str, transform_branches: List[FlowBranch], neo4j_conn=None, vault=None):
super().__init__(sketch_id, scan_id, neo4j_conn=neo4j_conn, vault=vault)
self.transform_branches = transform_branches
self.scanners = {} # Map of nodeId -> scanner instance
self.execution_log_file = None # Path to the execution log file
self._create_execution_log()
self._load_scanners()
def _create_execution_log(self) -> None:
"""
Create the initial execution log JSON file with transform configuration.
"""
try:
# Create a directory for storing transform files if it doesn't exist
transform_dir = "transform_logs"
os.makedirs(transform_dir, exist_ok=True)
# Create filename with sketch_id and scan_id
filename = f"transform_execution_{self.sketch_id}_{self.scan_id}.json"
self.execution_log_file = os.path.join(transform_dir, filename)
# Serialize the transform branches
serialized_branches = to_json_serializable(self.transform_branches)
# Create initial log structure
initial_log = {
"sketch_id": self.sketch_id,
"scan_id": self.scan_id,
"created_at": datetime.now().isoformat(),
"updated_at": datetime.now().isoformat(),
"status": "initialized",
"transform_branches": serialized_branches,
"execution_log": [],
"summary": {
"total_steps": 0,
"completed_steps": 0,
"failed_steps": 0,
"total_execution_time_ms": 0
},
"final_results": {}
}
# Count total steps
total_steps = 0
for branch in self.transform_branches:
for step in branch.steps:
if step.type != "type":
total_steps += 1
initial_log["summary"]["total_steps"] = total_steps
# Save initial log file
with open(self.execution_log_file, 'w', encoding='utf-8') as f:
json.dump(initial_log, f, indent=2, ensure_ascii=False)
Logger.info(self.sketch_id, {"message": f"Transform execution log created at {self.execution_log_file}"})
except Exception as e:
Logger.error(self.sketch_id, {"message": f"Failed to create execution log: {str(e)}"})
self.execution_log_file = None
def _update_execution_log(self, step_entry: Dict[str, Any], status: str = None) -> None:
"""
Update the execution log file with a new step entry or status update.
"""
if not self.execution_log_file:
return
try:
# Read current log
with open(self.execution_log_file, 'r', encoding='utf-8') as f:
log_data = json.load(f)
# Update timestamp
log_data["updated_at"] = datetime.now().isoformat()
# Update status if provided
if status:
log_data["status"] = status
# Add step entry if provided
if step_entry:
log_data["execution_log"].append(step_entry)
# Update summary
if step_entry["status"] == "completed":
log_data["summary"]["completed_steps"] += 1
elif step_entry["status"] == "error":
log_data["summary"]["failed_steps"] += 1
if "execution_time_ms" in step_entry:
log_data["summary"]["total_execution_time_ms"] += step_entry["execution_time_ms"]
# Write updated log
with open(self.execution_log_file, 'w', encoding='utf-8') as f:
json.dump(log_data, f, indent=2, ensure_ascii=False)
except Exception as e:
Logger.error(self.sketch_id, {"message": f"Failed to update execution log: {str(e)}"})
def _finalize_execution_log(self, final_results: Dict[str, Any]) -> None:
"""
Finalize the execution log with final results and status.
"""
if not self.execution_log_file:
return
try:
# Read current log
with open(self.execution_log_file, 'r', encoding='utf-8') as f:
log_data = json.load(f)
# Update final data
log_data["updated_at"] = datetime.now().isoformat()
log_data["status"] = "completed"
log_data["final_results"] = to_json_serializable(final_results)
# Write final log
with open(self.execution_log_file, 'w', encoding='utf-8') as f:
json.dump(log_data, f, indent=2, ensure_ascii=False)
Logger.info(self.sketch_id, {"message": f"Transform execution log finalized at {self.execution_log_file}"})
except Exception as e:
Logger.error(self.sketch_id, {"message": f"Failed to finalize execution log: {str(e)}"})
def _save_transform_branches(self) -> None:
"""
Save the transform branches to a JSON file for debugging and persistence.
"""
try:
# Create a directory for storing transform files if it doesn't exist
transform_dir = "transform_logs"
os.makedirs(transform_dir, exist_ok=True)
# Create filename with sketch_id and scan_id
filename = f"transform_branches_{self.sketch_id}_{self.scan_id}.json"
filepath = os.path.join(transform_dir, filename)
# Serialize the transform branches
serialized_branches = to_json_serializable(self.transform_branches)
# Save to JSON file
with open(filepath, 'w', encoding='utf-8') as f:
json.dump({
"sketch_id": self.sketch_id,
"scan_id": self.scan_id,
"timestamp": datetime.now().isoformat(),
"transform_branches": serialized_branches
}, f, indent=2, ensure_ascii=False)
Logger.info(self.sketch_id, {"message": f"Transform branches saved to {filepath}"})
except Exception as e:
Logger.error(self.sketch_id, {"message": f"Failed to save transform branches: {str(e)}"})
def _load_scanners(self) -> None:
if not self.transform_branches:
raise ValueError("No transform branches provided")
@@ -42,7 +197,16 @@ class TransformOrchestrator(Scanner):
if not ScannerRegistry.scanner_exists(scanner_name):
raise ValueError(f"Scanner '{scanner_name}' not found in registry")
scanner = ScannerRegistry.get_scanner(scanner_name, self.sketch_id, self.scan_id, neo4j_conn=self.neo4j_conn)
# Pass the step params to the scanner instance
scanner_params = node.params if hasattr(node, 'params') and node.params else {}
scanner = ScannerRegistry.get_scanner(
scanner_name,
self.sketch_id,
self.scan_id,
neo4j_conn=self.neo4j_conn,
vault=self.vault,
params=scanner_params
)
self.scanners[node_id] = scanner
def resolve_reference(self, ref_value: str, results_mapping: Dict[str, Any]) -> Any:
@@ -56,8 +220,8 @@ class TransformOrchestrator(Scanner):
def prepare_scanner_inputs(self, step: FlowStep, results_mapping: Dict[str, Any], initial_values: List[str]) -> List[Any]:
"""
Prépare les inputs d'un scanner à partir des références et des résultats précédents.
Gère les références simples, les listes, et les valeurs directes.
Prepare the inputs for a scanner based on the references and previous results.
Handles single references, lists, and direct values.
"""
inputs = {}
@@ -139,6 +303,9 @@ class TransformOrchestrator(Scanner):
"""
The actual async implementation of the scan logic
"""
# Update execution log to indicate scan has started
self._update_execution_log(None, "running")
results = {
"initial_values": values,
"branches": [],
@@ -178,17 +345,39 @@ class TransformOrchestrator(Scanner):
continue
scanner_name = scanner.name()
step_start_time = time.time()
step_result = {
"nodeId": node_id,
"scanner": scanner_name,
"status": "error" # Default to error, will update on success
}
# Create execution log entry
log_entry = {
"step_id": f"{branch_id}_{node_id}",
"branch_id": branch_id,
"branch_name": branch_name,
"node_id": node_id,
"scanner_name": scanner_name,
"inputs": to_json_serializable(scanner_inputs),
"outputs": None,
"status": "running",
"error": None,
"timestamp": datetime.now().isoformat(),
"execution_time_ms": 0
}
try:
# Update status for current step
completed_steps += 1
if not scanner_inputs:
step_result["error"] = "No inputs available"
error_msg = "No inputs available"
step_result["error"] = error_msg
log_entry["status"] = "error"
log_entry["error"] = error_msg
log_entry["execution_time_ms"] = int((time.time() - step_start_time) * 1000)
self._update_execution_log(log_entry)
branch_results["steps"].append(step_result)
continue
@@ -196,18 +385,25 @@ class TransformOrchestrator(Scanner):
cache_key = f"{node_id}:{str(scanner_inputs)}"
if cache_key in scanner_results_cache:
outputs = scanner_results_cache[cache_key]
log_entry["cache_hit"] = True
else:
# Execute the scanner
outputs = scanner.execute(scanner_inputs)
outputs = await scanner.execute(scanner_inputs)
if not isinstance(outputs, (dict, list)):
raise ValueError(f"Scanner '{scanner_name}' returned unsupported output format")
# Cache the results
scanner_results_cache[cache_key] = outputs
log_entry["cache_hit"] = False
# Store the outputs in the step result
step_result["outputs"] = outputs
# Store the outputs in the step result (serialize to avoid JSON issues)
step_result["outputs"] = to_json_serializable(outputs)
step_result["status"] = "completed"
# Update log entry with success
log_entry["outputs"] = to_json_serializable(outputs)
log_entry["status"] = "completed"
log_entry["execution_time_ms"] = int((time.time() - step_start_time) * 1000)
# Update the global results mapping with the outputs
self.update_results_mapping(outputs, step.outputs, results_mapping)
# Also store the raw outputs in the main results
@@ -217,16 +413,27 @@ class TransformOrchestrator(Scanner):
except ValidationError as e:
error_msg = f"Validation error: {str(e)}"
Logger.error(self.sketch_id, {"message": error_msg})
step_result["error"] = error_msg
log_entry["status"] = "error"
log_entry["error"] = error_msg
log_entry["execution_time_ms"] = int((time.time() - step_start_time) * 1000)
results["results"][node_id] = {"error": error_msg}
self._update_execution_log(log_entry)
return results
except Exception as e:
error_msg = f"Error during scan: {str(e)}"
Logger.error(self.sketch_id, {"message":error_msg})
step_result["error"] = error_msg
log_entry["status"] = "error"
log_entry["error"] = error_msg
log_entry["execution_time_ms"] = int((time.time() - step_start_time) * 1000)
results["results"][node_id] = {"error": error_msg}
self._update_execution_log(log_entry)
return results
# Update execution log with this step
self._update_execution_log(log_entry)
branch_results["steps"].append(step_result)
results["branches"].append(branch_results)
@@ -236,17 +443,8 @@ class TransformOrchestrator(Scanner):
# Include the final reference mapping for debugging
results["reference_mapping"] = results_mapping
# Finalize execution log
self._finalize_execution_log(results)
return results
def results_to_json(self, results: Any) -> Any:
if isinstance(results, UUID):
return str(results)
if isinstance(results, datetime):
return results.isoformat()
if isinstance(results, BaseModel):
return results.dict()
if isinstance(results, list):
return [self.results_to_json(item) for item in results]
if isinstance(results, dict):
return {key: self.results_to_json(value) for key, value in results.items()}
return results
+7 -2
View File
@@ -13,7 +13,7 @@ from app.scanners.ips.cidr_to_ips import CidrToIpsScanner
from app.scanners.organizations.org_to_asn import OrgToAsnScanner
from app.scanners.domains.domain_to_asn import DomainToAsnScanner
from app.scanners.crypto.wallet_to_transactions import CryptoWalletAddressToTransactions
from app.scanners.crypto.wallet_to_nfts import WalletAddressToNFTs
from app.scanners.crypto.wallet_to_nfts import CryptoWalletAddressToNFTs
from app.scanners.domains.to_website import DomainToWebsiteScanner
from app.scanners.websites.to_crawler import WebsiteToCrawler
from app.scanners.websites.to_domain import WebsiteToDomainScanner
@@ -53,6 +53,8 @@ class ScannerRegistry:
"module": scanner.__module__,
"doc": scanner.__doc__,
"key": scanner.key(),
"params": scanner.get_params_schema(),
"requires_key": scanner.requires_key(),
}
for name, scanner in cls._scanners.items()
}
@@ -72,6 +74,9 @@ class ScannerRegistry:
"category": category,
"inputs": scanner.input_schema(),
"outputs": scanner.output_schema(),
"params": {},
"params_schema": scanner.get_params_schema(),
"requires_key": scanner.requires_key(),
})
return scanners_by_category
@@ -88,7 +93,7 @@ ScannerRegistry.register(CidrToIpsScanner)
ScannerRegistry.register(OrgToAsnScanner)
ScannerRegistry.register(DomainToAsnScanner)
ScannerRegistry.register(CryptoWalletAddressToTransactions)
ScannerRegistry.register(WalletAddressToNFTs)
ScannerRegistry.register(CryptoWalletAddressToNFTs)
ScannerRegistry.register(DomainToWebsiteScanner)
ScannerRegistry.register(WebsiteToCrawler)
ScannerRegistry.register(WebsiteToDomainScanner)
+14 -3
View File
@@ -1,18 +1,20 @@
import os
import uuid
from dotenv import load_dotenv
from typing import List
from typing import List, Optional
from celery import states
from app.core.celery import celery
from app.scanners.orchestrator import TransformOrchestrator
from app.core.postgre_db import SessionLocal, get_db
from app.core.graph_db import Neo4jConnection
from app.core.vault import Vault
from app.types.transform import FlowBranch
from app.models.models import Scan
from sqlalchemy.orm import Session
from app.core.logger import Logger
from app.core.enums import EventLevel
from app.utils import to_json_serializable
load_dotenv()
URI = os.getenv("NEO4J_URI_BOLT")
@@ -25,7 +27,7 @@ db: Session = next(get_db())
logger = Logger()
@celery.task(name="run_transform", bind=True)
def run_scan(self, transform_branches, values: List[str], sketch_id: str | None):
def run_scan(self, transform_branches, values: List[str], sketch_id: str | None, owner_id: Optional[str] = None):
session = SessionLocal()
try:
@@ -42,19 +44,28 @@ def run_scan(self, transform_branches, values: List[str], sketch_id: str | None)
session.add(scan)
session.commit()
# Create vault instance if owner_id is provided
vault = None
if owner_id:
try:
vault = Vault(session, uuid.UUID(owner_id))
except Exception as e:
Logger.error(sketch_id, {"message": f"Failed to create vault: {str(e)}"})
transform_branches = [FlowBranch(**branch) for branch in transform_branches]
scanner = TransformOrchestrator(
sketch_id=sketch_id,
scan_id=str(scan_id),
transform_branches=transform_branches,
neo4j_conn=neo4j_connection,
vault=vault,
)
# Use the synchronous scan method which internally handles the async operations
results = scanner.scan(values=values)
scan.status = EventLevel.COMPLETED
scan.results = scanner.results_to_json(results)
scan.results = to_json_serializable(results)
session.commit()
return {"result": scan.results}
+3 -1
View File
@@ -50,8 +50,10 @@ class SubfinderTool(DockerTool):
def is_installed(self) -> bool:
return super().is_installed()
def launch(self, domain: str, args:List[str]) -> Any:
def launch(self, domain: str, args: List[str] = None) -> Any:
subdomains: set[str] = set()
if args is None:
args = []
command = f"-d {domain} {' '.join(args)}"
result = super().launch(command)
for sub in result.split("\n"):
+1
View File
@@ -17,6 +17,7 @@ class Edge(BaseModel):
class FlowStep(BaseModel):
nodeId: str = Field(..., description="ID of the associated node", title="Node ID")
params: Optional[Dict[str, Any]] = Field(None, description="Parameters for the step", title="Parameters")
type: Literal["type", "scanner"] = Field(..., description="Type of step - either type transformation or scanner", title="Step Type")
inputs: Dict[str, Any] = Field(..., description="Input data for this step", title="Inputs")
outputs: Dict[str, Any] = Field(..., description="Output data from this step", title="Outputs")
+37 -4
View File
@@ -7,10 +7,8 @@ from urllib.parse import urlparse
import re
import ssl
import socket
from typing import Dict, Any, Type
from typing import Any, Dict
from pydantic import BaseModel
from typing import Dict, Any, List, Optional, Type
from pydantic import BaseModel, ValidationError, Field, create_model
def is_valid_ip(address: str) -> bool:
try:
@@ -254,3 +252,38 @@ def flatten(data_dict, prefix=""):
key = f"{prefix}{key}"
flattened[key] = value
return flattened
def get_inline_relationships(nodes: List[Any], edges: List[Any]) -> List[str]:
"""
Get the inline relationships for a list of nodes and edges.
"""
relationships = []
for edge in edges:
source = next((node for node in nodes if node["id"] == edge["source"]), None)
target = next((node for node in nodes if node["id"] == edge["target"]), None)
if source and target:
relationships.append({"source":source, "edge":edge, "target":target})
return relationships
def to_json_serializable(obj):
"""Convert any object to a JSON-serializable format."""
import json
from pydantic import BaseModel
try:
# Test if already JSON serializable
json.dumps(obj)
return obj
except (TypeError, ValueError):
# Handle common cases
if isinstance(obj, BaseModel):
# Use mode='json' to ensure all Pydantic types are properly serialized
return obj.model_dump(mode='json') if hasattr(obj, 'model_dump') else obj.dict()
elif isinstance(obj, (list, tuple)):
return [to_json_serializable(item) for item in obj]
elif isinstance(obj, dict):
return {key: to_json_serializable(value) for key, value in obj.items()}
else:
# Convert anything else to string
return str(obj)
+6
View File
@@ -0,0 +1,6 @@
[tool:pytest]
asyncio_mode = auto
testpaths = tests
python_files = test_*.py
python_functions = test_*
addopts = -v -s
+3 -1
View File
@@ -18,6 +18,7 @@ neo4j
faker
sqlalchemy
psycopg2-binary
asyncpg
alembic==1.13.0
passlib[bcrypt]
sse-starlette
@@ -30,4 +31,5 @@ spacy
docker
git+https://github.com/soxoj/maigret
git+https://github.com/reconurge/recontrack.git
git+https://github.com/reconurge/reconcrawl.git
git+https://github.com/reconurge/reconcrawl.git
pytest-asyncio
@@ -1,13 +1,14 @@
from app.scanners.crypto.wallet_to_transactions import WalletAddressToTransactions
from app.types.wallet import Wallet, WalletTransaction
import pytest
from app.scanners.crypto.wallet_to_transactions import CryptoWalletAddressToTransactions
from app.types.wallet import CryptoWallet, CryptoWalletTransaction
scanner = WalletAddressToTransactions("sketch_123", "scan_123")
scanner = CryptoWalletAddressToTransactions("sketch_123", "scan_123", params={"ETHERSCAN_API_KEY": "ta-clef-api"},)
def test_wallet_address_to_transactions_name():
assert scanner.name() == "wallet_to_transactions"
def test_wallet_address_to_transactions_category():
assert scanner.category() == "crypto"
assert scanner.category() == "CryptoCryptoWallet"
def test_wallet_address_to_transactions_key():
assert scanner.key() == "address"
@@ -16,47 +17,68 @@ def test_preprocess_with_string():
input_data = ["0x742d35Cc6634C0532925a3b844Bc454e4438f44e"]
result = scanner.preprocess(input_data)
assert len(result) == 1
assert isinstance(result[0], Wallet)
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
def test_preprocess_with_dict():
input_data = [{"address": "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"}]
result = scanner.preprocess(input_data)
assert len(result) == 1
assert isinstance(result[0], Wallet)
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
def test_preprocess_with_wallet_object():
wallet = Wallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")
wallet = CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")
input_data = [wallet]
result = scanner.preprocess(input_data)
assert len(result) == 1
assert isinstance(result[0], Wallet)
assert isinstance(result[0], CryptoWallet)
assert result[0].address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
def test_scan_mocked_transactions(monkeypatch):
# Mock the _get_transactions method
def mock_get_transactions(address):
@pytest.mark.asyncio
async def test_scan_mocked_transactions(monkeypatch):
# Mock the _get_transactions method - note it takes address and api_key parameters
async def mock_get_transactions(address, api_key):
return [
WalletTransaction(
CryptoWalletTransaction(
hash="0x123",
from_address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e",
to_address="0x456",
value=1000000000000000000, # 1 ETH in wei
timestamp=1234567890
source=CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e"),
target=CryptoWallet(address="0x456"),
value=1.0, # 1 ETH
timestamp="1234567890",
block_number="12345",
block_hash="0xabc",
nonce="1",
transaction_index="0",
gas="21000",
gas_price="20000000000",
gas_used="21000",
cumulative_gas_used="21000",
input="0x",
contract_address=None
)
]
monkeypatch.setattr(scanner, "_get_transactions", mock_get_transactions)
input_data = [Wallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")]
result = scanner.scan(input_data)
input_data = [CryptoWallet(address="0x742d35Cc6634C0532925a3b844Bc454e4438f44e")]
result = await scanner.scan(input_data)
assert len(result) == 1
assert len(result[0]) == 1
assert result[0][0].hash == "0x123"
assert result[0][0].from_address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
assert result[0][0].to_address == "0x456"
assert result[0][0].value == 1000000000000000000
assert result[0][0].timestamp == 1234567890
assert result[0][0].source.address == "0x742d35Cc6634C0532925a3b844Bc454e4438f44e"
assert result[0][0].target.address == "0x456"
assert result[0][0].value == 1.0
assert result[0][0].timestamp == "1234567890"
def test_scanner_requires_api_key():
"""Test that the scanner validates required ETHERSCAN_API_KEY parameter at construction"""
with pytest.raises(ValueError, match="Scanner wallet_to_transactions received invalid params"):
CryptoWalletAddressToTransactions("sketch_123", "scan_123", params={})
def test_scanner_with_invalid_api_key_type():
"""Test that the scanner validates parameter types"""
with pytest.raises(ValueError, match="Scanner wallet_to_transactions received invalid params"):
CryptoWalletAddressToTransactions("sketch_123", "scan_123", params={"ETHERSCAN_API_KEY": 123})
+101
View File
@@ -0,0 +1,101 @@
import sys
import os
import asyncio
import uuid
import random
# Add the parent directory to Python path so we can import from app
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
from app.scanners.crypto.wallet_to_transactions import CryptoWalletAddressToTransactions
from app.scanners.crypto.wallet_to_nfts import CryptoWalletAddressToNFTs
from app.core.vault import Vault
from app.core.postgre_db import SessionLocal
from app.models.models import Key, Profile
def setup_test_data(db_session, owner_id: uuid.UUID):
"""Set up test data including user and API keys"""
# Generate a unique email for testing
unique_email = f"test_{random.randint(1000, 9999)}@example.com"
# First, create a test user/profile
test_user = Profile(
id=owner_id,
email=unique_email,
hashed_password="test_hashed_password",
is_active=True
)
db_session.add(test_user)
db_session.commit() # Commit the user first
# Now create a test API key for ETHERSCAN_API_KEY
test_key = Key(
id=uuid.uuid4(),
name="vaultKey1", # This matches the vaultRef in scanner params
owner_id=owner_id,
encrypted_key="test_etherscan_api_key_12345" # Mock API key
)
db_session.add(test_key)
db_session.commit()
print(f"Created test user with email: {test_user.email}")
print(f"Created test API key with name: {test_key.name}")
return test_key
async def main():
# Create database session using the existing get_db pattern
db_session = SessionLocal()
try:
# Create a test owner ID (in real app this would be from authentication)
owner_id = uuid.uuid4()
# Set up test data
test_key = setup_test_data(db_session, owner_id)
# Create vault instance
vault = Vault(db=db_session, owner_id=owner_id)
# Create scanners with vault
scanner = CryptoWalletAddressToTransactions(
sketch_id="sketch_123",
scan_id="scan_123",
vault=vault, # Pass the vault
params={"ETHERSCAN_API_KEY": "vaultKey1doesntexist",
"ETHERSCAN_API_URL": "https://api.etherscan.io/api"}
)
scanner2 = CryptoWalletAddressToNFTs(
sketch_id="sketch_123",
scan_id="scan_123",
vault=vault, # Pass the vault
params={"ETHERSCAN_API_KEY": "vaultKey1doesntexist",
"ETHERSCAN_API_URL": "https://api.etherscan.io/api"}
)
print("Scanner 1 params (before async_init):", scanner.get_params())
print("Scanner 2 params (before async_init):", scanner2.get_params())
# Initialize scanners asynchronously
await scanner.async_init()
await scanner2.async_init()
print("Scanner 1 params (after async_init):", scanner.get_params())
print("Scanner 2 params (after async_init):", scanner2.get_params())
# Test vault directly
secret = vault.get_secret("vaultKey1")
print(f"Retrieved secret from vault: {secret}")
# Test vault with non-existent key
secret_none = vault.get_secret("nonexistent")
print(f"Retrieved secret for non-existent key: {secret_none}")
finally:
db_session.close()
if __name__ == "__main__":
asyncio.run(main())
+41
View File
@@ -0,0 +1,41 @@
from app.scanners.base import build_params_model
def test_build_params_model_valid():
param_schema = [
{
"name": "ETHERSCAN_API_KEY",
"type": "string",
"description": "The Etherscan API key to use for the transaction lookup.",
"required": True
},
{
"name": "url",
"type": "string",
"description": "Base URL for API",
"required": False,
"default": "https://api.etherscan.io/api"
}
]
ParamsModel = build_params_model(param_schema)
validated_params = ParamsModel(ETHERSCAN_API_KEY="clef-123")
assert validated_params.ETHERSCAN_API_KEY == "clef-123"
assert validated_params.url == "https://api.etherscan.io/api"
def test_build_params_model_invalid():
param_schema = [
{
},
{
"name": "url",
"type": "string",
"description": "Base URL for API",
"required": False,
"default": "https://api.etherscan.io/api"
}
]
ParamsModel = build_params_model(param_schema)
validated_params = ParamsModel(ETHERSCAN_API_KEY="clef-123")
assert validated_params.ETHERSCAN_API_KEY == "clef-123"
assert validated_params.url == "https://api.etherscan.io/api"
@@ -0,0 +1 @@
<?xml version="1.0"?><!DOCTYPE svg PUBLIC '-//W3C//DTD SVG 1.1//EN' 'http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd'><svg height="512px" style="enable-background:new 0 0 512 512;" version="1.1" viewBox="0 0 512 512" width="512px" xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"><g id="_x33_63-visa_x2C__Credit_card"><g><path d="M485.999,115.445v281.108c0,21.165-17.17,38.335-38.333,38.335H64.334 c-21.163,0-38.333-17.17-38.333-38.335V115.445c0-21.164,17.17-38.334,38.333-38.334h383.332 C468.829,77.111,485.999,94.282,485.999,115.445z" style="fill:#0353A5;"/><g><path d="M421.791,192.111h-24.837c-7.666,0-13.498,2.237-16.771,10.302l-47.678,113.8h33.702 c0,0,5.511-15.332,6.708-18.606h41.209c0.956,4.392,3.832,18.606,3.832,18.606h29.709L421.791,192.111z M382.179,272.211 c2.637-7.106,12.778-34.738,12.778-34.738c-0.16,0.239,2.636-7.268,4.233-11.9l2.235,10.702c0,0,6.07,29.708,7.428,35.937 H382.179z" style="fill:#FFFFFF;"/><path d="M164.32,192.111l-31.385,84.653l-3.435-17.171l-11.181-57.021 c-1.837-7.906-7.507-10.142-14.535-10.461H52.116l-0.56,2.476c12.618,3.195,23.879,7.827,33.701,13.656l28.591,107.813h33.94 l50.471-123.944H164.32z" style="fill:#FFFFFF;"/><polygon points="211.197,192.111 191.152,316.214 223.178,316.214 243.302,192.111 " style="fill:#FFFFFF;"/><path d="M307.989,241.865c-11.26-5.67-18.128-9.504-18.128-15.333c0.159-5.271,5.829-10.702,18.448-10.702 c10.461-0.24,18.128,2.237,23.877,4.711l2.876,1.358l4.392-26.833c-6.31-2.477-16.371-5.272-28.75-5.272 c-31.704,0-53.985,16.932-54.145,41.049c-0.24,17.809,15.971,27.713,28.11,33.701c12.379,6.068,16.611,10.063,16.611,15.413 c-0.159,8.307-10.063,12.14-19.247,12.14c-12.777,0-19.646-1.998-30.105-6.627l-4.234-1.999l-4.472,27.871 c7.505,3.435,21.402,6.47,35.777,6.631c33.701,0.078,55.663-16.614,55.902-42.328 C335.063,261.511,326.438,250.729,307.989,241.865z" style="fill:#FFFFFF;"/></g></g></g><g id="Layer_1"/></svg>

After

Width:  |  Height:  |  Size: 1.9 KiB

@@ -0,0 +1,11 @@
import { fetchWithAuth } from './api'
export const KeyService = {
get: () => fetchWithAuth(`/api/keys`, { method: 'GET' }),
getById: (key_id: string) => fetchWithAuth(`/api/keys/${key_id}`, { method: 'GET' }),
create: (data: { name: string; key: string }) => fetchWithAuth(`/api/keys/create`, {
method: 'POST',
body: JSON.stringify(data)
}),
deleteById: (key_id: string) => fetchWithAuth(`/api/keys/${key_id}`, { method: 'DELETE' }),
}
@@ -12,8 +12,8 @@ export const sketchService = {
method: 'GET',
});
},
getGraphDataById: async (sketchId: string): Promise<any> => {
return fetchWithAuth(`/api/sketches/${sketchId}/graph`, {
getGraphDataById: async (sketchId: string, inline: boolean = false): Promise<any> => {
return fetchWithAuth(`/api/sketches/${sketchId}/graph?format=${inline ? 'inline' : ''}`, {
method: 'GET',
});
},
@@ -1,12 +0,0 @@
import { fetchWithAuth } from './api'
export const ThirdPartyKeysService = {
get: () => fetchWithAuth(`/api/third_party_keys`, { method: 'GET' }),
getServices: () => fetchWithAuth(`/api/third_party_keys/services`, { method: 'GET' }),
getById: (key_id: string) => fetchWithAuth(`/api/third_party_keys/${key_id}`, { method: 'GET' }),
create: (data: { service: string; key: string }) => fetchWithAuth(`/api/third_party_keys/create`, {
method: 'POST',
body: JSON.stringify(data)
}),
deleteById: (key_id: string) => fetchWithAuth(`/api/third_party_keys/${key_id}`, { method: 'DELETE' }),
}
@@ -2,7 +2,7 @@ import { useChat } from '@/hooks/use-chat'
import { memo, useEffect, useState, useRef } from 'react'
import { ChatPanel, ContextList } from './chat-prompt'
import { Button } from '../ui/button'
import { Sparkles, X, Plus, History } from 'lucide-react'
import { X, Plus, History } from 'lucide-react'
import { useKeyboardShortcut } from '@/hooks/use-keyboard-shortcut'
import { Card } from '../ui/card'
import { useLayoutStore } from '@/stores/layout-store'
@@ -127,7 +127,7 @@ function FloatingChat() {
<>
<div className="flex items-center justify-between p-3 border-b">
<div className="flex w-full items-center justify-between gap-1">
<div className='flex items-center gap-2'>
<div className='flex items-center gap-2 truncate text-ellipsis'>
<Button
variant="ghost"
size="icon"
@@ -138,7 +138,7 @@ function FloatingChat() {
>
<Plus className="h-3 w-3" />
</Button>
<span className='text-sm opacity-60'>{chat?.title}</span>
<span className='text-sm opacity-60 truncate text-ellipsis'>{chat?.title}</span>
</div>
<div className='flex items-center gap-2'>
<Button
@@ -5,6 +5,7 @@ import remarkGfm from 'remark-gfm'
import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter'
import { dracula } from 'react-syntax-highlighter/dist/esm/styles/prism';
import { cn } from '@/lib/utils';
import { CopyButton } from '../copy';
function parseMarkdownIntoBlocks(markdown: string): string[] {
const tokens = marked.lexer(markdown);
@@ -18,14 +19,19 @@ const MemoizedMarkdownBlock = memo(
code({ node, className, children, ...props }) {
const match = /language-(\w+)/.exec(className || '')
return match ? (
<SyntaxHighlighter
children={String(children).replace(/\n$/, '')}
style={dracula}
language={match[1]}
PreTag="div"
className="rounded border border-border bg-muted px-1 py-0.5 text-sm"
{...props}
/>
<div className='relative'>
<SyntaxHighlighter
children={String(children).replace(/\n$/, '')}
style={dracula}
language={match[1]}
PreTag="div"
className="rounded border border-border bg-muted px-1 py-0.5 text-sm"
{...props}
/>
<div className='absolute top-2 right-2'>
<CopyButton content={String(children).replace(/\n$/, '')} />
</div>
</div>
) : (
<code className={cn("rounded border border-border bg-muted px-1 py-0.5 text-sm", className)} {...props}>
{children}
@@ -60,7 +60,7 @@ export const DraggableItem = memo(function DraggableItem({
onDragStart={handleDragStart}
onDragEnd={handleDragEnd}
className={cn(
"px-0 py-1 group flex flex-col items-center w-full justify-center gap-1 rounded-md relative overflow-hidden",
"px-3 py-2 group flex items-center w-full gap-3 text-ellipsis rounded-md relative overflow-hidden border-1 border-l-3 transition-all hover:bg-accent/50",
{
"opacity-50": isDragging || disabled,
"cursor-not-allowed": disabled,
@@ -69,11 +69,12 @@ export const DraggableItem = memo(function DraggableItem({
)}
style={{ borderLeftColor: colorStr }}
>
<div className="flex justify-center items-center bg-background w-full text-left h-20 border rounded-lg">
<IconComponent size={24} color={colorStr} type={type} />
<div className="flex justify-center items-center w-8 h-8 rounded-md bg-background border">
<IconComponent size={16} color="#6b7280" type={type} />
</div>
<div className="space-y-1 truncate flex-1">
<h3 className="text-xs font-normal opacity-60 truncate">{label}</h3>
<div className="w-full p-1 text-left flex-1 min-w-0">
<h3 className="text-sm font-medium truncate w-full">{label}</h3>
<p className="text-xs opacity-60 truncate w-full">{description}</p>
</div>
</button>
</TooltipTrigger>
@@ -17,6 +17,8 @@ import { toast } from 'sonner'
import MapPanel from '../map/map-panel'
import NewActions from './new-actions'
const GraphReactForce = lazy(() => import('./graph-react-force'))
const RelationshipsTable = lazy(() => import('@/components/table/relationships-view'))
const Graph = lazy(() => import('./graph'))
const NODE_COUNT_THRESHOLD = 500;
@@ -136,12 +138,14 @@ const GraphPanel = ({ graphData, isLoading, isRefetching }: GraphPanelProps) =>
<>{view === "table" && <NodesTable />}
{["force", "hierarchy"].includes(view) && <Graph />}
{view === "map" && <MapPanel />}
{view === "relationships" && <RelationshipsTable />}
</>
) : (<>
{view === "force" && <GraphReactForce />}
{view === "hierarchy" && <WallEditor isRefetching={isRefetching} isLoading={loading} />}
{view === "table" && <NodesTable />}
{view === "map" && <MapPanel />}
{view === "relationships" && <RelationshipsTable />}
</>)}
</Suspense>
{/* <Graph /> */}
@@ -12,7 +12,19 @@ interface GraphReactForceProps {
style?: React.CSSProperties;
}
interface LabelBounds {
x: number;
y: number;
width: number;
height: number;
nodeId: string;
nodeSize: number;
}
const NODE_COUNT_THRESHOLD = 10;
const ZOOM_MIN = 0.3;
const ZOOM_INTERVAL = 2;
const ZOOM_MAX = 10;
const GraphReactForce: React.FC<GraphReactForceProps> = () => {
const nodes = useGraphStore(s => s.nodes) as GraphNode[];
@@ -22,12 +34,11 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {
const colors = useNodesDisplaySettings(s => s.colors) as Record<ItemType, string>;
const toggleNodeSelection = useGraphStore(s => s.toggleNodeSelection);
const clearSelectedNodes = useGraphStore(s => s.clearSelectedNodes);
// const currentNode = useGraphStore(s => s.currentNode);
const setActions = useGraphControls(s => s.setActions);
const [menu, setMenu] = useState<any>(null);
const [currentZoom, setCurrentZoom] = useState(1);
const shouldUseSimpleRendering = useMemo(() => {
// Use simple rendering for large node counts or when zoomed out
return nodes.length > NODE_COUNT_THRESHOLD || currentZoom < 2.5;
}, [nodes.length, currentZoom]);
@@ -40,7 +51,8 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {
let nodeLabel = node.data.label;
if (!nodeLabel && 'caption' in node) nodeLabel = (node as any).caption;
if (!nodeLabel) nodeLabel = node.id;
const randMin = ZOOM_MIN + Math.random() * (ZOOM_MAX - ZOOM_MIN - 1);
const randMax = randMin + ZOOM_INTERVAL;
return {
...node,
nodeLabel: nodeLabel,
@@ -48,8 +60,10 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {
nodeSize: size,
nodeType: type,
val: size,
randMin: randMin,
randMax: randMax
};
});
}).sort((a, b) => b.nodeSize - a.nodeSize); // Sort by size, largest first
const edgeGroups = new Map();
rawEdges.forEach(edge => {
@@ -91,7 +105,6 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {
setMenu(null)
}, [clearSelectedNodes, setMenu]);
const handleZoomIn = useCallback((graph: any) => {
const zoom = graph.zoom();
graph.zoom(zoom * 1.5);
@@ -106,57 +119,207 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {
graph.zoomToFit(400);
}, []);
const renderNode = useCallback((node: any, ctx: CanvasRenderingContext2D, globalScale: number) => {
const size = node.nodeSize;
const type = node.nodeType as ItemType;
// Helper functions for rendering
const drawNodeCircle = useCallback((ctx: CanvasRenderingContext2D, node: any, size: number) => {
ctx.beginPath();
ctx.arc(node.x, node.y, size * 0.65, 0, 2 * Math.PI);
ctx.fillStyle = node.nodeColor;
ctx.fill();
}, []);
// Use simple rendering for large node counts
if (shouldUseSimpleRendering) {
// Simple circle rendering for large node counts
ctx.beginPath();
ctx.arc(node.x, node.y, size * .65, 0, 2 * Math.PI);
ctx.fillStyle = node.nodeColor;
ctx.fill();
} else {
// Full rendering with icon and label
ctx.beginPath();
// ctx.arc(node.x, node.y, size / 2, 0, 2 * Math.PI);
ctx.fillStyle = node.nodeColor;
ctx.fill();
const img = new Image();
img.src = `/icons/${type}.svg`;
// Draw icon if available
ctx.drawImage(img, node.x - size / 2, node.y - size / 2, size, size);
const drawNodeIcon = useCallback((ctx: CanvasRenderingContext2D, node: any, size: number, type: ItemType) => {
const img = new Image();
img.src = `/icons/${type}.svg`;
ctx.drawImage(img, node.x - size / 2, node.y - size / 2, size, size);
}, []);
if (globalScale > 3) {
const label = node.nodeLabel || node.label || node.id;
if (label) {
const fontSize = Math.max(2, size * 0.4);
ctx.font = `${fontSize}px -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif`;
const textWidth = ctx.measureText(label).width;
const padding = 2;
const shouldShowLabel = useCallback((globalScale: number, randMin: number, randMax: number) => {
return globalScale >= randMin && globalScale <= randMax;
}, []);
ctx.fillStyle = 'rgba(255, 255, 255, 0.45)';
ctx.strokeStyle = 'rgba(0, 0, 0, 0.05)';
ctx.lineWidth = 0.2;
const bgWidth = textWidth + padding * 2;
const bgHeight = fontSize + padding;
const bgX = node.x - bgWidth / 2;
const bgY = node.y + size / 2 + 1;
// Check if two rectangles overlap
const doLabelsOverlap = useCallback((bounds1: LabelBounds, bounds2: LabelBounds) => {
return !(bounds1.x + bounds1.width < bounds2.x ||
bounds2.x + bounds2.width < bounds1.x ||
bounds1.y + bounds1.height < bounds2.y ||
bounds2.y + bounds2.height < bounds1.y);
}, []);
ctx.beginPath();
ctx.roundRect(bgX, bgY, bgWidth, bgHeight, .75);
ctx.fill();
ctx.stroke();
const drawLabelBackground = useCallback((ctx: CanvasRenderingContext2D, node: any, label: string, fontSize: number) => {
const textWidth = ctx.measureText(label).width;
const padding = 6;
const bgWidth = textWidth + padding * 2;
const bgHeight = fontSize + padding;
const bgX = node.x - bgWidth / 2;
const bgY = node.y + node.nodeSize / 2 + 1;
ctx.fillStyle = theme === "dark" ? 'rgb(255, 255, 255)' : 'rgb(0, 0, 0)';
ctx.textAlign = 'center';
ctx.textBaseline = 'middle';
ctx.fillText(label, node.x, bgY + bgHeight / 2);
ctx.fillStyle = 'rgba(149, 149, 149, 0.56)';
ctx.strokeStyle = 'rgba(0, 0, 0, 0.05)';
ctx.lineWidth = 0.2;
ctx.beginPath();
ctx.roundRect(bgX, bgY, bgWidth, bgHeight, 4);
ctx.fill();
ctx.stroke();
return { bgX, bgY, bgHeight, bgWidth };
}, []);
const drawLabel = useCallback((ctx: CanvasRenderingContext2D, node: any, label: string, fontSize: number, bgY: number, bgHeight: number) => {
ctx.fillStyle = theme === "dark" ? 'rgb(255, 255, 255)' : 'rgb(0, 0, 0)';
ctx.textAlign = 'center';
ctx.textBaseline = 'middle';
ctx.fillText(label, node.x, bgY + bgHeight / 2);
}, [theme]);
// Efficient collision detection using spatial partitioning
const collisionGridRef = React.useRef<Map<string, LabelBounds[]>>(new Map());
const renderedLabelsRef = React.useRef<Set<string>>(new Set());
const gridSize = 50;
// Get all grid keys that a label bounds might overlap
const getOverlappingGridKeys = useCallback((bounds: LabelBounds) => {
const keys = new Set<string>();
const startX = Math.floor(bounds.x / gridSize);
const endX = Math.floor((bounds.x + bounds.width) / gridSize);
const startY = Math.floor(bounds.y / gridSize);
const endY = Math.floor((bounds.y + bounds.height) / gridSize);
for (let x = startX; x <= endX; x++) {
for (let y = startY; y <= endY; y++) {
keys.add(`${x},${y}`);
}
}
return Array.from(keys);
}, []);
// Check for collisions using spatial partitioning with size-based priority
const checkCollision = useCallback((newBounds: LabelBounds) => {
const gridKeys = getOverlappingGridKeys(newBounds);
for (const key of gridKeys) {
const boundsInGrid = collisionGridRef.current.get(key) || [];
for (const existingBounds of boundsInGrid) {
if (doLabelsOverlap(newBounds, existingBounds)) {
// If the new label is from a larger node, it takes priority
if (newBounds.nodeSize > existingBounds.nodeSize) {
// Remove the smaller node's label from the grid
const updatedBounds = boundsInGrid.filter(b => b.nodeId !== existingBounds.nodeId);
collisionGridRef.current.set(key, updatedBounds);
renderedLabelsRef.current.delete(existingBounds.nodeId);
continue; // Check other bounds in this grid
} else {
// New label is from a smaller node, so it gets rejected
return true;
}
}
}
}
}, [shouldUseSimpleRendering, theme]);
return false;
}, [getOverlappingGridKeys, doLabelsOverlap]);
// Add bounds to spatial grid
const addBoundsToGrid = useCallback((bounds: LabelBounds) => {
const gridKeys = getOverlappingGridKeys(bounds);
for (const key of gridKeys) {
if (!collisionGridRef.current.has(key)) {
collisionGridRef.current.set(key, []);
}
collisionGridRef.current.get(key)!.push(bounds);
}
}, [getOverlappingGridKeys]);
// Custom renderer with efficient collision detection
const renderNodeWithCollisionDetection = useCallback((node: any, ctx: CanvasRenderingContext2D, globalScale: number) => {
const type = node.nodeType as ItemType;
const label = node.nodeLabel || node.label || node.id;
// Always render the node itself
if (shouldUseSimpleRendering) {
drawNodeCircle(ctx, node, node.nodeSize);
} else {
drawNodeIcon(ctx, node, node.nodeSize, type);
}
// Check if we should show a label
if (!shouldShowLabel(globalScale, node.randMin, node.randMax)) return;
// Determine font size based on zoom level
let fontSize: number;
if (shouldUseSimpleRendering) {
fontSize = Math.max(8, 14 / globalScale);
} else {
if (globalScale > 3) {
fontSize = Math.max(2, node.nodeSize * 0.4);
} else {
fontSize = Math.max(8, 14 / globalScale);
}
}
// Set font for measurement
ctx.font = `${fontSize}px -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif`;
// Measure text to check for collisions
const textWidth = ctx.measureText(label).width;
const padding = 2;
const bgWidth = textWidth + padding * 2;
const bgHeight = fontSize + padding;
const bgX = node.x - bgWidth / 2;
const bgY = node.y + node.nodeSize / 2 + 1;
const newLabelBounds: LabelBounds = {
x: bgX,
y: bgY,
width: bgWidth,
height: bgHeight,
nodeId: node.id,
nodeSize: node.nodeSize
};
// Check if this label was previously rendered
const wasPreviouslyRendered = renderedLabelsRef.current.has(node.id);
// If not previously rendered, check for collisions
if (!wasPreviouslyRendered) {
if (checkCollision(newLabelBounds)) {
return; // Don't render this label
}
// Add this label to the spatial grid
addBoundsToGrid(newLabelBounds);
// Mark as rendered
renderedLabelsRef.current.add(node.id);
}
// Render the label background and text
if (shouldUseSimpleRendering || globalScale > 3) {
// Draw background for detailed labels
const { bgY: finalBgY, bgHeight: finalBgHeight } = drawLabelBackground(ctx, node, label, fontSize);
drawLabel(ctx, node, label, fontSize, finalBgY, finalBgHeight);
} else {
// Draw simple label without background
ctx.fillStyle = theme === "dark" ? 'rgb(255, 255, 255)' : 'rgb(0, 0, 0)';
ctx.textAlign = 'center';
ctx.textBaseline = 'middle';
ctx.fillText(label, node.x, node.y);
}
}, [shouldUseSimpleRendering, drawNodeCircle, drawNodeIcon, shouldShowLabel, checkCollision, addBoundsToGrid, drawLabelBackground, drawLabel, theme]);
// Reset collision grid and rendered labels only when nodes change
useEffect(() => {
collisionGridRef.current.clear();
renderedLabelsRef.current.clear();
}, [nodes.length, nodes.map(n => n.id).join(',')]); // Also reset when node IDs change
// Reset only when zoom changes significantly
const lastZoomRef = React.useRef(currentZoom);
useEffect(() => {
const zoomDiff = Math.abs(currentZoom - lastZoomRef.current);
if (zoomDiff > 0.5) {
collisionGridRef.current.clear();
renderedLabelsRef.current.clear();
lastZoomRef.current = currentZoom;
}
}, [currentZoom]);
// Set actions in store
useEffect(() => {
@@ -194,19 +357,15 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {
(data: any, event: MouseEvent) => {
if (!containerRef.current) return;
const pane = containerRef.current.getBoundingClientRect();
// Use the mouse event coordinates instead of node position
const relativeX = event.clientX - pane.left;
const relativeY = event.clientY - pane.top;
// Calculate available space in each direction
const menuWidth = 320; // Default menu width
const menuHeight = 250; // Use a more reasonable height for overflow calculation
const padding = 20; // Minimum padding from edges
const menuWidth = 320;
const menuHeight = 250;
const padding = 20;
// Determine if menu would overflow in each direction
const wouldOverflowRight = relativeX + menuWidth + padding > pane.width;
const wouldOverflowBottom = relativeY + menuHeight + padding > pane.height;
// Calculate final position
let finalTop = 0;
let finalLeft = 0;
let finalRight = 0;
@@ -244,6 +403,7 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {
return (
<div className="relative h-full grow w-full bg-background">
{/* {currentZoom} */}
<div ref={containerRef} style={{ width: '100%', height: '100%', minHeight: 300, minWidth: 300 }}>
<ForceGraph2D
ref={graphRef}
@@ -253,6 +413,7 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {
nodeLabel="label"
nodeColor={node => shouldUseSimpleRendering ? node.nodeColor : "#00000000"}
nodeRelSize={6}
enableNodeDrag={false}
linkDirectionalArrowLength={3.5}
linkDirectionalArrowRelPos={1}
cooldownTicks={100}
@@ -270,7 +431,7 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {
}}
d3AlphaDecay={0.02}
d3VelocityDecay={0.3}
nodeCanvasObject={renderNode}
nodeCanvasObject={renderNodeWithCollisionDetection}
onNodeHover={(node: any) => {
if (node) {
graphData.nodes.forEach((n: any) => {
@@ -295,6 +456,7 @@ const GraphReactForce: React.FC<GraphReactForceProps> = () => {
backgroundColor="transparent"
onZoom={(zoom) => setCurrentZoom(zoom.k)}
/>
{menu && <ContextMenu
onClick={handleBackgroundClick}
{...menu}
@@ -71,7 +71,7 @@ CosmographLoader.displayName = "CosmographLoader"
const GRAPH_CONFIG = {
// simulationCenter: 0.2,
useQuadtree: true,
simulationDecay: 150,
simulationDecay: 100,
simulationGravity: 0.25,
// simulationLinkSpring: 2,
simulationRepulsion: 2,
@@ -48,19 +48,19 @@ export const ItemsPanel = memo(function LeftPanel() {
/>
</div>
</div>
<div className="grid grid-cols-1 gap-3">
<div className="flex flex-col gap-3">
{isLoading ? <SkeletonList rowCount={8} /> :
actionItems?.map((item: ActionItem) => {
if (item.children && item.children.length > 0) {
return (
<div key={item.id} className="@container">
<div key={item.id}>
<div className="py-1 text-sm text-muted-foreground">
<div className="flex items-center gap-2">
<span>{item.label} ({item.children.length})</span>
{item.comingSoon && <span className="ml-1 text-xs text-muted-foreground">(Soon)</span>}
</div>
</div>
<div className="grid grid-cols-2 @xs:grid-cols-3 @sm:grid-cols-4 @md:grid-cols-5 gap-2">
<div className="flex flex-col gap-2">
{item.children.map((childItem) => (
<DraggableItem
key={childItem.id}
@@ -15,7 +15,8 @@ import {
GitFork,
Waypoints,
Table,
MapPin
MapPin,
List
} from "lucide-react"
import { memo, useCallback } from "react"
import { sketchService } from "@/api/sketch-service"
@@ -121,6 +122,10 @@ export const Toolbar = memo(function Toolbar({ isLoading }: { isLoading: boolean
setView("map")
}, [setView])
const handleRelationshipsLayout = useCallback(() => {
setView("relationships")
}, [setView])
const handleDagreLayoutTB = useCallback(() => {
setView("hierarchy")
onLayout && onLayout("dagre-tb")
@@ -195,12 +200,16 @@ export const Toolbar = memo(function Toolbar({ isLoading }: { isLoading: boolean
tooltip={"Table view"}
onClick={handleTableLayout}
/>
<ToolbarButton
icon={<List className="h-4 w-4 opacity-70" />}
tooltip={"Relationships view"}
onClick={handleRelationshipsLayout}
/>
<ToolbarButton
icon={<MapPin className="h-4 w-4 opacity-70" />}
tooltip={"Map view"}
onClick={handleMapLayout}
/>
{/* <ToolbarButton
icon={<Rotate3D className="h-4 w-4 opacity-70" />}
tooltip={"3D Graph"}
@@ -0,0 +1,55 @@
// import { useCallback, useRef } from 'react';
// export interface ForceGraphControls {
// zoomIn: () => void;
// zoomOut: () => void;
// zoomToFit: () => void;
// resetZoom: () => void;
// }
// export const useForceGraphControls = () => {
// const graphRef = useRef<any>();
// const setGraphRef = useCallback((ref: any) => {
// graphRef.current = ref;
// }, []);
// const zoomIn = useCallback(() => {
// if (graphRef.current) {
// const zoom = graphRef.current.zoom();
// graphRef.current.zoom(zoom * 1.5);
// }
// }, []);
// const zoomOut = useCallback(() => {
// if (graphRef.current) {
// const zoom = graphRef.current.zoom();
// graphRef.current.zoom(zoom * 0.75);
// }
// }, []);
// const zoomToFit = useCallback(() => {
// if (graphRef.current) {
// graphRef.current.zoomToFit(400);
// }
// }, []);
// const resetZoom = useCallback(() => {
// if (graphRef.current) {
// graphRef.current.zoom(1);
// graphRef.current.centerAt(0, 0);
// }
// }, []);
// const controls: ForceGraphControls = {
// zoomIn,
// zoomOut,
// zoomToFit,
// resetZoom,
// };
// return {
// controls,
// setGraphRef,
// };
// };
@@ -6,12 +6,10 @@ import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
import { FileCode2, Search, Info, Star } from 'lucide-react';
import { Transform } from '@/types';
import { GraphNode, useGraphStore } from '@/stores/graph-store';
import { GraphNode } from '@/stores/graph-store';
import { useLaunchTransform } from '@/hooks/use-launch-transform';
import { useParams } from '@tanstack/react-router';
import { capitalizeFirstLetter, cn } from '@/lib/utils';
import { useConfirm } from '@/components/use-confirm-dialog';
import { useLayoutStore } from '@/stores/layout-store';
import NodeActions from '../../node-actions';
export default function ContextMenu({
@@ -114,7 +112,7 @@ export default function ContextMenu({
{...props}
>
{/* Header with title and action buttons */}
<div className="px-3 py-2 border-b border-border flex items-center justify-between flex-shrink-0">
<div className="px-3 py-2 border-b gap-1 border-border flex items-center justify-between flex-shrink-0">
<div className='flex text-xs items-center gap-1 truncate'>
<span className='block truncate'>{node.data.label}</span> - <span className='block'>{node.data.type}</span>
</div>
@@ -35,9 +35,9 @@ const nodeTypes = {
custom: CustomNode,
};
const edgeTypes = {
custom: SimpleFloatingEdge,
};
// const edgeTypes = {
// custom: SimpleFloatingEdge,
// };
const Wall = memo(({ theme, edges }: { theme: ColorMode, edges: GraphEdge[] }) => {
const { fitView, zoomIn, zoomOut, setCenter } = useReactFlow()
@@ -47,6 +47,7 @@ const Wall = memo(({ theme, edges }: { theme: ColorMode, edges: GraphEdge[] }) =
const setActions = useGraphControls(state => state.setActions)
// Memoize store selectors
const nodes = useGraphStore(state => state.nodes)
const toggleNodeSelection = useGraphStore(state => state.toggleNodeSelection)
const currentNode = useGraphStore(state => state.currentNode)
const setCurrentNode = useGraphStore(state => state.setCurrentNode)
const clearSelectedNodes = useGraphStore(state => state.clearSelectedNodes)
@@ -140,7 +141,11 @@ const Wall = memo(({ theme, edges }: { theme: ColorMode, edges: GraphEdge[] }) =
)
const onNodeClick: NodeMouseHandler = useCallback(
(_: React.MouseEvent, node: Node) => {
(e: React.MouseEvent, node: Node) => {
if (e.shiftKey) {
toggleNodeSelection(node as GraphNode, true)
return
}
const typedNode = node as GraphNode
setCurrentNode(typedNode)
setActiveTab("entities")
@@ -0,0 +1,34 @@
import { useQuery } from "@tanstack/react-query"
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
import { KeyService } from "@/api/key-service"
import { type Key as KeyType } from '@/types/key'
export default function KeySelector({ onChange, value }: { onChange: (value: KeyType) => void, value?: KeyType }) {
// Fetch keys
const { data: keys = [], isLoading } = useQuery<KeyType[]>({
queryKey: ['keys'],
queryFn: () => KeyService.get(),
})
const handleValueChange = (keyId: string) => {
const selectedKey = keys.find(key => key.id === keyId)
if (selectedKey) {
onChange(selectedKey)
}
}
return (
<Select onValueChange={handleValueChange} value={value?.id || ""} disabled={isLoading}>
<SelectTrigger className="w-full">
<SelectValue placeholder="Select a key" />
</SelectTrigger>
<SelectContent>
{keys.map((key: KeyType) => (
<SelectItem key={key.id} value={key.id}>
{key.name}
</SelectItem>
))}
</SelectContent>
</Select>
)
}
@@ -1,4 +1,3 @@
"use client"
import { useState } from "react"
import {
Popover,
@@ -29,7 +29,7 @@ export function Sidebar() {
{ icon: Home, label: "Dashboard", href: "/dashboard/" },
{ icon: Fingerprint, label: "Cases", href: "/dashboard/investigations" },
// { icon: BrickWall, label: "Walls", href: "/dashboard/investigations/wall" },
{ icon: Workflow, label: "Transforms", href: "/dashboard/transforms" },
{ icon: Workflow, label: "Flows", href: "/dashboard/transforms" },
]
const commonClasses =
@@ -1,4 +1,3 @@
"use client"
import { useState } from "react"
import {
Popover,
@@ -33,10 +33,10 @@ export function StatusBar() {
<span>Vault</span>
</Button>
</Link>
<Button variant="ghost" size="sm" className="h-6 gap-1 text-xs">
{/* <Button variant="ghost" size="sm" className="h-6 gap-1 text-xs">
<Trash strokeWidth={1.4} className="h-3 w-3 opacity-60" />
<span>Trash</span>
</Button>
</Button> */}
<Legend />
<InfoDialog />
<ModeToggle />
@@ -18,6 +18,7 @@ import { cn } from "@/lib/utils";
interface DataTableProps<TData, TValue> {
columns: ColumnDef<TData, TValue>[];
data: TData[];
noStickyCol?: boolean;
}
// Stable style objects to prevent re-renders
@@ -45,7 +46,8 @@ const VirtualRow = React.memo<{
virtualRow: any;
lastCellIndex: number;
columnSizeVars: Record<string, number>;
}>(({ row, virtualRow, lastCellIndex, columnSizeVars }) => {
noStickyCol?: boolean;
}>(({ row, virtualRow, lastCellIndex, columnSizeVars, noStickyCol }) => {
const visibleCells = row.getVisibleCells();
// Memoize the row style with transform
@@ -70,6 +72,7 @@ const VirtualRow = React.memo<{
isBeforeLast={index === lastCellIndex - 1}
isLast={index === lastCellIndex}
columnSizeVars={columnSizeVars}
noStickyCol={noStickyCol}
/>
))}
</tr>
@@ -84,7 +87,8 @@ const VirtualCell = React.memo<{
isLast: boolean;
isBeforeLast: boolean,
columnSizeVars: Record<string, number>;
}>(({ cell, index, isFirst, isLast, isBeforeLast, columnSizeVars }) => {
noStickyCol?: boolean;
}>(({ cell, index, isFirst, isLast, isBeforeLast, columnSizeVars, noStickyCol }) => {
const cellStyle = React.useMemo(() => ({
width: `calc(var(--col-${cell.column.id}-size) * 1px)`,
minWidth: `calc(var(--col-${cell.column.id}-size) * 1px)`,
@@ -97,9 +101,9 @@ const VirtualCell = React.memo<{
const cellClassName = React.useMemo(() => cn(
"px-4 py-1 truncate relative overflow-hidden",
!isBeforeLast && !isLast && "border-r",
index === 0 && "sticky z-10 left-0 bg-card border-b border-r",
index === 1 && "sticky z-11 left-[50px] bg-card border-b border-r",
isLast && "sticky right-0 z-10 bg-background px-0 border-l border-b",
index === 0 && !noStickyCol && "sticky z-10 left-0 bg-card border-b border-r",
index === 1 && !noStickyCol && "sticky z-11 left-[50px] bg-card border-b border-r",
isLast && !noStickyCol && "sticky right-0 z-10 bg-background px-0 border-l border-b",
), [isFirst, isLast, index, isBeforeLast]);
return (
@@ -114,10 +118,11 @@ const VirtualCell = React.memo<{
});
// Table body component
const TableBody = ({ table, tableContainerRef, columnSizeVars }: {
const TableBody = ({ table, tableContainerRef, columnSizeVars, noStickyCol }: {
table: Table<any>;
tableContainerRef: React.RefObject<HTMLDivElement>;
columnSizeVars: Record<string, number>;
noStickyCol?: boolean;
}) => {
const { rows } = table.getRowModel();
@@ -168,6 +173,7 @@ const TableBody = ({ table, tableContainerRef, columnSizeVars }: {
virtualRow={virtualRow}
lastCellIndex={lastCellIndex}
columnSizeVars={columnSizeVars}
noStickyCol={noStickyCol}
/>
);
})}
@@ -207,7 +213,8 @@ const HeaderCell = React.memo<{
isLast: boolean;
isBeforeLast: boolean;
columnSizeVars: Record<string, number>;
}>(({ header, index, isFirst, isLast, isBeforeLast, columnSizeVars }) => {
noStickyCol?: boolean;
}>(({ header, index, isFirst, isLast, isBeforeLast, columnSizeVars, noStickyCol }) => {
const headerStyle = React.useMemo(() => ({
width: `calc(var(--header-${header.id}-size) * 1px)`,
minWidth: `calc(var(--header-${header.id}-size) * 1px)`,
@@ -217,10 +224,10 @@ const HeaderCell = React.memo<{
const headerClassName = React.useMemo(() => cn(
!isBeforeLast && !isLast && "border-r",
"px-4 py-1 text-center font-medium text-muted-foreground border-b overflow-hidde",
isFirst && `!sticky z-30`,
index === 0 && "left-0 bg-card",
index === 1 && "left-[50px] bg-card",
isLast && "!sticky right-0 z-30 bg-background border-l",
isFirst && !noStickyCol && `!sticky z-30`,
index === 0 && !noStickyCol && "left-0 bg-card",
index === 1 && !noStickyCol && "left-[50px] bg-card",
isLast && !noStickyCol && "!sticky right-0 z-30 bg-background border-l",
), [isFirst, isLast, index, isBeforeLast]);
return (
@@ -246,6 +253,7 @@ const HeaderCell = React.memo<{
export function DataTable<TData, TValue>({
columns,
data,
noStickyCol = false,
}: DataTableProps<TData, TValue>) {
const tableContainerRef = React.useRef<HTMLDivElement>(null);
const [sorting, setSorting] = React.useState<SortingState>([]);
@@ -311,6 +319,7 @@ export function DataTable<TData, TValue>({
isFirst={index === 0 || index === 1}
isLast={index === lastColumnIndex}
columnSizeVars={columnSizeVars}
noStickyCol={noStickyCol}
/>
))}
</tr>
@@ -320,6 +329,7 @@ export function DataTable<TData, TValue>({
table={table}
tableContainerRef={tableContainerRef}
columnSizeVars={columnSizeVars}
noStickyCol={noStickyCol}
/>
</table>
</div>
@@ -0,0 +1,286 @@
import { useQuery } from "@tanstack/react-query";
import { useParams } from "@tanstack/react-router";
import { sketchService } from "@/api/sketch-service";
import { GraphNode } from "@/stores/graph-store";
import { useVirtualizer } from "@tanstack/react-virtual";
import { useRef, useState, useMemo } from "react";
import { Input } from "@/components/ui/input";
import { Badge } from "@/components/ui/badge";
import { Card, CardContent } from "@/components/ui/card";
import { Search, ArrowRight, Users, Link, Filter } from "lucide-react";
import { Skeleton } from "@/components/ui/skeleton";
import { useIcon } from "@/hooks/use-icon";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
export type RelationshipType = {
source: GraphNode
target: GraphNode
edge: { label: string }
}
const ITEM_HEIGHT = 76; // Balanced spacing between items
// Separate component for relationship item to avoid hook order issues
interface RelationshipItemProps {
relationship: RelationshipType;
style: React.CSSProperties;
}
function RelationshipItem({ relationship, style }: RelationshipItemProps) {
const SourceIcon = useIcon(relationship.source.data?.type, relationship.source.data?.src);
const TargetIcon = useIcon(relationship.target.data?.type, relationship.target.data?.src);
return (
<div style={style} className="px-4 pb-2">
<Card className="h-[64px] hover:shadow-md transition-shadow duration-200 p-0">
<CardContent className="p-3 h-[64px] flex items-center gap-2 min-w-0">
{/* Source Node */}
<div className="flex items-center gap-2 flex-1 min-w-0">
<div className="flex items-center justify-center w-8 h-8 rounded-full bg-muted flex-shrink-0">
<SourceIcon className="h-4 w-4" />
</div>
<div className="flex-1 min-w-0">
<p className="font-medium truncate text-sm">
{relationship.source.data?.label || relationship.source.id}
</p>
<Badge variant="outline" className="text-xs truncate max-w-full">
<span className="truncate">
{relationship.source.data?.type || 'unknown'}
</span>
</Badge>
</div>
</div>
{/* Relationship Arrow */}
<div className="flex items-center gap-2 px-2 flex-shrink-0 min-w-0 max-w-[200px]">
<div className="flex items-center gap-2 bg-muted/50 px-2 py-1 rounded-full min-w-0">
<Badge variant="secondary" className="text-xs font-medium truncate max-w-[120px]">
<span className="truncate">
{relationship.edge.label}
</span>
</Badge>
<ArrowRight className="h-4 w-4 text-muted-foreground flex-shrink-0" />
</div>
</div>
{/* Target Node */}
<div className="flex items-center gap-2 flex-1 justify-end min-w-0">
<div className="flex-1 min-w-0 text-right">
<p className="font-medium truncate text-sm">
{relationship.target.data?.label || relationship.target.id}
</p>
<Badge variant="outline" className="text-xs truncate max-w-full">
<span className="truncate">
{relationship.target.data?.type || 'unknown'}
</span>
</Badge>
</div>
<div className="flex items-center justify-center w-8 h-8 rounded-full bg-muted flex-shrink-0">
<TargetIcon className="h-4 w-4" />
</div>
</div>
</CardContent>
</Card>
</div>
);
}
export default function RelationshipsTable() {
const { id: sketchId } = useParams({ from: "/_auth/dashboard/investigations/$investigationId/$type/$id" })
const { data: relationships, isLoading } = useQuery({
queryKey: ["graph", 'relationships_view', sketchId],
enabled: Boolean(sketchId),
queryFn: () => sketchService.getGraphDataById(sketchId as string, true),
});
const [searchQuery, setSearchQuery] = useState("");
const [selectedType, setSelectedType] = useState<string>("all");
const parentRef = useRef<HTMLDivElement>(null);
// Filter relationships based on search and type
const filteredRelationships = useMemo(() => {
if (!relationships) return [];
return relationships.filter((rel: RelationshipType) => {
const matchesSearch = searchQuery === "" ||
rel.source.data?.label?.toLowerCase().includes(searchQuery.toLowerCase()) ||
rel.target.data?.label?.toLowerCase().includes(searchQuery.toLowerCase()) ||
rel.edge.label?.toLowerCase().includes(searchQuery.toLowerCase());
const matchesType = selectedType === "all" ||
rel.source.data?.type === selectedType ||
rel.target.data?.type === selectedType;
return matchesSearch && matchesType;
});
}, [relationships, searchQuery, selectedType]);
// Get unique node types for filter
const nodeTypes = useMemo(() => {
if (!relationships) return [];
const types = new Set<string>();
relationships.forEach((rel: RelationshipType) => {
if (rel.source.data?.type) types.add(rel.source.data.type);
if (rel.target.data?.type) types.add(rel.target.data.type);
});
return Array.from(types).sort();
}, [relationships]);
const virtualizer = useVirtualizer({
count: filteredRelationships.length,
getScrollElement: () => parentRef.current,
estimateSize: () => ITEM_HEIGHT,
overscan: 5,
});
if (isLoading) {
return (
<div className="w-full pt-18 space-y-4 p-12">
{/* Header with stats */}
<div className="flex items-center justify-between">
<div className="space-y-1">
<h2 className="text-2xl font-bold tracking-tight">Relationships</h2>
<p className="text-muted-foreground">
<Skeleton className="h-4 w-32 inline-block" />
</p>
</div>
<Badge variant="secondary" className="flex items-center gap-2">
<Users className="h-4 w-4" />
<Skeleton className="h-4 w-8" />
</Badge>
</div>
{/* Search and Filter */}
<div className="flex gap-4">
<div className="relative flex-1">
<Search className="absolute left-3 top-1/2 transform -translate-y-1/2 h-4 w-4 text-muted-foreground" />
<Input
placeholder="Search relationships, nodes, or types..."
value={searchQuery}
onChange={(e) => setSearchQuery(e.target.value)}
className="pl-10"
disabled
/>
</div>
<Select value={selectedType} onValueChange={setSelectedType} disabled>
<SelectTrigger className="w-48">
<Filter className="h-4 w-4 mr-2 text-muted-foreground" />
<SelectValue placeholder="Filter by type" />
</SelectTrigger>
<SelectContent>
<SelectItem value="all">All Types</SelectItem>
</SelectContent>
</Select>
</div>
{/* Skeleton List */}
<div className="grow overflow-auto py-4 rounded-lg border">
<div className="space-y-2 px-4">
{Array.from({ length: 8 }).map((_, i) => (
<Skeleton key={i} className="h-[64px] w-full" />
))}
</div>
</div>
</div>
);
}
if (!relationships || relationships.length === 0) {
return (
<div className="w-full pt-18 flex items-center justify-center h-96">
<div className="text-center space-y-4">
<Link className="mx-auto h-12 w-12 text-muted-foreground" />
<div>
<h3 className="text-lg font-semibold">No relationships found</h3>
<p className="text-muted-foreground">This sketch doesn't have any relationships yet.</p>
</div>
</div>
</div>
);
}
return (
<div className="w-full grow flex flex-col pt-18 space-y-4 p-4 px-6">
{/* Header with stats */}
<div className="flex items-center justify-between">
<div className="space-y-1">
<h2 className="text-2xl font-bold tracking-tight">Relationships</h2>
<p className="text-muted-foreground">
{filteredRelationships.length} of {relationships.length} relationships
</p>
</div>
<Badge variant="secondary" className="flex items-center gap-2">
<Users className="h-4 w-4" />
{relationships.length} total
</Badge>
</div>
{/* Search and Filter */}
<div className="flex gap-4">
<div className="relative flex-1">
<Search className="absolute left-3 top-1/2 transform -translate-y-1/2 h-4 w-4 text-muted-foreground" />
<Input
placeholder="Search relationships, nodes, or types..."
value={searchQuery}
onChange={(e) => setSearchQuery(e.target.value)}
className="pl-10"
/>
</div>
<Select value={selectedType} onValueChange={setSelectedType}>
<SelectTrigger className="w-48">
<Filter className="h-4 w-4 mr-2 text-muted-foreground" />
<SelectValue placeholder="Filter by type" />
</SelectTrigger>
<SelectContent>
<SelectItem value="all">All Types</SelectItem>
{nodeTypes.map((type) => (
<SelectItem key={type} value={type}>
<span className="capitalize">{type}</span>
</SelectItem>
))}
</SelectContent>
</Select>
</div>
{/* Virtualized List */}
<div
ref={parentRef}
className="grow overflow-auto py-4 rounded-lg border"
>
<div
style={{
height: `${virtualizer.getTotalSize()}px`,
width: '100%',
position: 'relative',
}}
className="space-y-2"
>
{virtualizer.getVirtualItems().map((virtualRow) => {
const relationship = filteredRelationships[virtualRow.index];
return (
<RelationshipItem
key={virtualRow.index}
relationship={relationship}
style={{
position: 'absolute',
top: 0,
left: 0,
width: '100%',
height: `${virtualRow.size}px`,
transform: `translateY(${virtualRow.start}px)`,
}}
/>
);
})}
</div>
</div>
</div>
);
}
@@ -19,7 +19,8 @@ import {
import "@xyflow/react/dist/style.css"
import { TrashIcon, Play, Pause, SkipForward, RefreshCw } from "lucide-react"
import { Button } from "@/components/ui/button"
import ScannerNode, { type ScannerNodeData } from "./scanner-node"
import ScannerNode from "./scanner-node"
import { type ScannerNodeData } from "@/types/transform"
import { categoryColors } from "./scanner-data"
import { FlowControls } from "./controls"
import { getDagreLayoutedElements } from "@/lib/utils"
@@ -38,6 +39,7 @@ import {
SelectValue,
} from "@/components/ui/select"
import { useTheme } from "../theme-provider"
import ParamsDialog from "./params-dialog"
const nodeTypes: NodeTypes = {
scanner: ScannerNode,
@@ -153,6 +155,9 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform
inputs: scannerData.inputs,
outputs: scannerData.outputs,
doc: scannerData.doc,
requires_key: scannerData.requires_key,
params: scannerData.params,
params_schema: scannerData.params_schema
},
}
const updatedNodes = [...nodes, newNode]
@@ -260,11 +265,11 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform
[nodes, edges, transformId, router, setLoading],
)
const handleSaveTransform = useCallback(() => {
const handleSaveTransform = useCallback(async () => {
if (!transformId) {
setShowModal(true)
} else {
saveTransform(transform?.name || "", transform?.description || "")
await saveTransform(transform?.name || "", transform?.description || "")
}
}, [transformId, saveTransform, transform])
@@ -551,6 +556,7 @@ const TransformEditorFlow = memo(({ initialEdges, initialNodes, theme, transform
isSaved={Boolean(transformId)}
/>
<Background bgColor="var(--background)" />
<ParamsDialog />
<MiniMap className="bg-background" position="bottom-left" pannable zoomable />
{selectedNode && (
<NodePanel node={selectedNode} onDelete={handleDeleteNode} />
@@ -16,8 +16,8 @@ const NewTransform = ({ children }: { children: ReactNode }) => {
setNodes([])
setEdges([])
const response = await transformService.create(JSON.stringify({
name: "New Transform",
description: "A new transform",
name: "New flow",
description: "A new example flow.",
category: [],
transform_schema: {}
}))
@@ -0,0 +1,188 @@
import { useTransformStore } from '@/stores/transform-store'
import { DialogHeader, DialogFooter, Dialog, DialogContent, DialogTitle, DialogDescription } from '../ui/dialog'
import { Button } from '../ui/button'
import { useCallback, useState, useEffect } from 'react'
import { ScannerParamSchemaItem } from '@/types'
import { Label } from '../ui/label'
import { Input } from '../ui/input'
import KeySelector from '../keys/key-select'
import { type Key } from '@/types/key'
import { useQuery } from "@tanstack/react-query"
import { KeyService } from "@/api/key-service"
import { Tabs, TabsList, TabsTrigger, TabsContent } from '../ui/tabs'
const ParamsDialog = () => {
const openParamsDialog = useTransformStore(s => s.openParamsDialog)
const setOpenParamsDialog = useTransformStore(s => s.setOpenParamsDialog)
const selectedNode = useTransformStore(s => s.selectedNode)
const updateNode = useTransformStore(s => s.updateNode)
const [params, setParams] = useState<Record<string, string>>({})
const [settings, setSettings] = useState<Record<string, string>>({
duration: '30',
retry: '3',
timeout: '60',
priority: 'medium'
})
// Initialize params and settings when selectedNode changes
useEffect(() => {
if (selectedNode?.data.params) {
setParams(selectedNode.data.params)
}
if (selectedNode?.data.settings) {
setSettings({ ...settings, ...selectedNode.data.settings })
}
}, [selectedNode])
// Fetch keys to convert between IDs and Key objects
const { data: keys = [] } = useQuery<Key[]>({
queryKey: ['keys'],
queryFn: () => KeyService.get(),
})
const handleSave = useCallback(async () => {
if (!selectedNode) return
const updatedNode = {
...selectedNode,
data: {
...selectedNode.data,
params,
settings
}
}
updateNode(updatedNode)
setOpenParamsDialog(false)
}, [selectedNode, updateNode, params, settings, setOpenParamsDialog])
if (!selectedNode) return
return (
<Dialog open={openParamsDialog} onOpenChange={setOpenParamsDialog}>
<DialogContent className="sm:max-w-[725px]">
<DialogHeader>
<DialogTitle>Configure Transform</DialogTitle>
<DialogDescription>Configure parameters and settings for your transform.</DialogDescription>
</DialogHeader>
<Tabs defaultValue="parameters" className="w-full">
<TabsList className="grid w-full grid-cols-2">
<TabsTrigger value="parameters">Parameters</TabsTrigger>
<TabsTrigger value="settings">Settings</TabsTrigger>
</TabsList>
<TabsContent value="parameters" className="space-y-4 mt-4">
<div className="grid gap-4">
{selectedNode.data.params_schema.map((param: ScannerParamSchemaItem) => (
<div className="space-y-2" key={param.name}>
<div className="flex items-start flex-col">
<Label htmlFor={param.name} className="text-sm font-medium">
{param.name}
{param.required && <span className="text-destructive ml-1">*</span>}
</Label>
<p className='text-sm opacity-60'>{param.description}</p>
</div>
{param.type === 'vaultSecret' ? (
<KeySelector
onChange={(key) => setParams({ ...params, [param.name]: key.id })}
value={keys.find(key => key.id === params[param.name])}
/>
) : (
<Input
id={param.name}
type={param.type}
placeholder={param.default ?? param.name}
value={params[param.name] || ""}
onChange={(e) => setParams({ ...params, [param.name]: e.target.value })}
/>
)}
</div>
))}
</div>
</TabsContent>
<TabsContent value="settings" className="space-y-4 mt-4">
<div className="grid gap-4">
<div className="space-y-2">
<div className="flex items-start flex-col">
<Label htmlFor="duration" className="text-sm font-medium">
Duration (seconds)
</Label>
<p className='text-sm opacity-60'>Maximum execution time for this transform</p>
</div>
<Input
id="duration"
type="number"
placeholder="30"
value={settings.duration || ""}
onChange={(e) => setSettings({ ...settings, duration: e.target.value })}
/>
</div>
<div className="space-y-2">
<div className="flex items-start flex-col">
<Label htmlFor="retry" className="text-sm font-medium">
Retry Attempts
</Label>
<p className='text-sm opacity-60'>Number of retry attempts on failure</p>
</div>
<Input
id="retry"
type="number"
placeholder="3"
value={settings.retry || ""}
onChange={(e) => setSettings({ ...settings, retry: e.target.value })}
/>
</div>
<div className="space-y-2">
<div className="flex items-start flex-col">
<Label htmlFor="timeout" className="text-sm font-medium">
Timeout (seconds)
</Label>
<p className='text-sm opacity-60'>Connection timeout for network requests</p>
</div>
<Input
id="timeout"
type="number"
placeholder="60"
value={settings.timeout || ""}
onChange={(e) => setSettings({ ...settings, timeout: e.target.value })}
/>
</div>
<div className="space-y-2">
<div className="flex items-start flex-col">
<Label htmlFor="priority" className="text-sm font-medium">
Priority
</Label>
<p className='text-sm opacity-60'>Execution priority level</p>
</div>
<select
id="priority"
className="flex h-9 w-full rounded-md border border-input bg-transparent px-3 py-1 text-sm shadow-sm transition-colors file:border-0 file:bg-transparent file:text-sm file:font-medium placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:cursor-not-allowed disabled:opacity-50"
value={settings.priority || "medium"}
onChange={(e) => setSettings({ ...settings, priority: e.target.value })}
>
<option value="low">Low</option>
<option value="medium">Medium</option>
<option value="high">High</option>
</select>
</div>
</div>
</TabsContent>
</Tabs>
<DialogFooter>
<Button variant="outline" onClick={() => setOpenParamsDialog(false)}>
Cancel
</Button>
<Button onClick={handleSave}>
Save configuration
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
)
}
export default ParamsDialog
@@ -1,6 +1,7 @@
import { useMemo, useState } from "react"
import { Search, X } from "lucide-react"
import ScannerItem, { type Scanner } from "./scanner-item"
import ScannerItem from "./scanner-item"
import { type Scanner } from "@/types/transform"
import { Input } from "../ui/input"
import { Button } from "../ui/button"
import { transformService } from "@/api/transfrom-service"
@@ -1,12 +1,4 @@
import type { Scanner } from "./scanner-item"
export interface ScansData {
[category: string]: Scanner[]
}
export interface ScannerData {
items: ScansData
}
import type { Scanner, ScansData, ScannerData } from "@/types/transform"
// Default color for fallback cases
export const DEFAULT_COLOR = "#94a3b8"
@@ -3,37 +3,12 @@
import type React from "react"
import { memo, useCallback, useState } from "react"
import { Button } from "@/components/ui/button"
import { Info, GripVertical } from "lucide-react"
import { Info, GripVertical, KeySquare } from "lucide-react"
import { TooltipProvider } from "@/components/ui/tooltip"
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogTrigger } from "@/components/ui/dialog"
import { useNodesDisplaySettings } from "@/stores/node-display-settings"
// Types for the scanner based on the new structure
export interface ScannerProperty {
name: string
type: string
}
export interface ScannerIO {
type: string
properties: ScannerProperty[]
}
export interface Scanner {
class_name: string
category: string
name: string
module: string
doc: string | null
inputs: ScannerIO
outputs: ScannerIO
type: string
}
interface ScannerItemProps {
scanner: Scanner
category: string
}
import { Badge } from "../ui/badge"
import { type Scanner, type ScannerItemProps } from "@/types/transform"
// Custom equality function for ScannerItem
function areEqual(prevProps: ScannerItemProps, nextProps: ScannerItemProps) {
@@ -101,6 +76,10 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {
</DialogTrigger>
</div>
</div>
{scanner.requires_key &&
<div className="absolute bottom-3 right-3">
<KeySquare className="h-4 w-4 text-yellow-500" />
</div>}
</div>
<DialogContent className="sm:max-w-[725px] max-h-[90vh] overflow-y-auto">
<DialogHeader>
@@ -110,6 +89,11 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {
</DialogTitle>
</DialogHeader>
<div className="grid gap-4 py-4">
{scanner.requires_key &&
<Badge variant={"outline"} className=" top-3 right-3">
Key required <KeySquare className="h-4 w-4 text-yellow-500" />
</Badge>
}
<div className="space-y-2">
<h4 className="font-medium text-sm" style={{ color: borderInputColor }}>Description</h4>
<p className="text-sm text-muted-foreground">{scanner.doc || "No description available"}</p>
@@ -140,6 +124,9 @@ const ScannerItem = memo(({ scanner, category }: ScannerItemProps) => {
))}
</div>
</div>
<div className="space-y-2">
{JSON.stringify(scanner)}
</div>
</div>
</DialogContent>
</Dialog>
@@ -1,4 +1,4 @@
import { memo } from "react"
import { memo, useCallback } from "react"
import { Position } from "@xyflow/react"
import { LabeledHandle } from "../ui/labeled-handle"
import { Badge } from "../ui/badge"
@@ -6,38 +6,10 @@ import { BaseNodeSchema } from "../ui/base-node"
import { NodeStatusIndicator } from "../ui/node-status-indicator"
import { useNodesDisplaySettings } from "@/stores/node-display-settings"
import { useIcon } from "@/hooks/use-icon"
// Types for the scanner node based on the new structure
export interface ScannerNodeData extends Record<string, unknown> {
class_name: string
name: string
module: string
doc: string | null
category: string
type: string
color?: string
computationState?: "pending" | "processing" | "completed" | "error"
inputs: {
type: string
properties: Array<{
name: string
type: string
}>
}
outputs: {
type: string
properties: Array<{
name: string
type: string
}>
}
}
interface ScannerNodeProps {
data: ScannerNodeData
isConnectable?: boolean
selected?: boolean
}
import { cn } from "@/lib/utils"
import { KeySquare } from "lucide-react"
import { type ScannerNodeProps } from "@/types/transform"
import { TransformNode, useTransformStore } from "@/stores/transform-store"
// Custom equality function to prevent unnecessary re-renders
function areEqual(prevProps: ScannerNodeProps, nextProps: ScannerNodeProps) {
@@ -77,6 +49,11 @@ const ScannerNode = memo(({ data, selected, isConnectable }: ScannerNodeProps) =
const opacity = data.computationState === "pending" ? 0.5 : 1
const InputIconComponent = useIcon(data.inputs.type.toLowerCase() as string, null);
const OutputIconComponent = useIcon(data.outputs.type.toLowerCase() as string, null);
const setOpenParamsDialog = useTransformStore(s => s.setOpenParamsDialog)
const handleOpenParamsModal = useCallback(() => {
setOpenParamsDialog(true, data as unknown as TransformNode)
}, [setOpenParamsDialog, data])
const getStatusVariant = (state?: string) => {
@@ -95,30 +72,29 @@ const ScannerNode = memo(({ data, selected, isConnectable }: ScannerNodeProps) =
}
return (
<NodeStatusIndicator variant={getStatusVariant(data.computationState)}>
<NodeStatusIndicator variant={getStatusVariant(data.computationState)} showStatus={data.type !== "type"}>
<BaseNodeSchema
className="shadow-md rounded-md p-0 bg-background !max-w-[340px]"
className={cn("shadow-md relative p-0 bg-background ", data.type === "type" ? "rounded-full !max-w-[240px]" : "rounded-md !max-w-[340px]")}
style={{ borderLeftWidth: 5, borderRightWidth: 5, borderLeftColor: inputColor ?? outputColor, borderRightColor: outputColor, cursor: "grab", opacity }}
selected={selected}
>
<div className="p-3 bg-card rounded-t-md">
<div className="flex flex-col items-start gap-1 relative">
<div className="absolute top-0 right-0 flex items-center gap-2">
{/* <Badge variant={"outline"} className="">
{data.type}
</Badge> */}
{data.computationState && (
<Badge className={getStateColor(data.computationState)}>
{data.computationState}
</Badge>
)}
{data.type !== "type" &&
<div className="p-3 bg-card rounded-t-md">
<div className="flex flex-col items-start gap-1 relative">
<div className="absolute top-0 right-0 flex items-center gap-2">
{data.computationState && data.type !== "type" && (
<Badge className={getStateColor(data.computationState)}>
{data.computationState}
</Badge>
)}
</div>
<div className="font-semibold text-sm">{data.class_name}</div>
<p className="text-xs text-muted-foreground mt-2">{data.doc}</p>
</div>
<div className="font-semibold text-sm">{data.class_name}</div>
<p className="text-xs text-muted-foreground mt-2">{data.doc}</p>
</div>
</div>
}
<div>
<div className="grid grid-cols-2 py-1">
<div className={cn("grid grid-cols-2 py-1", data.type === "type" ? "grid-cols-1 p-2" : "grid-cols-2")}>
<div className="pl-0 pr-6">
{data.inputs.properties.length > 0 && (
<LabeledHandle
@@ -144,6 +120,11 @@ const ScannerNode = memo(({ data, selected, isConnectable }: ScannerNodeProps) =
)}
</div>
</div>
{data.requires_key &&
<Badge onClick={handleOpenParamsModal} variant={"outline"} className="absolute top-3 right-3">
Key required <KeySquare className="h-4 w-4 text-yellow-500" />
</Badge>
}
</div>
</BaseNodeSchema>
</NodeStatusIndicator>
@@ -14,7 +14,7 @@ const buttonVariants = cva(
destructive:
"bg-destructive text-white shadow-xs hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60",
outline:
"border shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-background/80 dark:hover:bg-background/90 backfrop-blur-sm",
"border shadow-xs hover:bg-accent hover:text-accent-foreground bg-background/80 hover:bg-background/90 backdrop-blur-sm",
secondary:
"bg-secondary text-secondary-foreground shadow-xs hover:bg-secondary/80",
ghost:
@@ -21,17 +21,20 @@ const nodeStatusIndicatorVariants = cva(
interface NodeStatusIndicatorProps extends VariantProps<typeof nodeStatusIndicatorVariants> {
className?: string
children?: React.ReactNode
showStatus: boolean
}
export function NodeStatusIndicator({
className,
variant,
children,
showStatus = true
}: NodeStatusIndicatorProps) {
return (
<div className="relative">
{children}
<div className={cn(nodeStatusIndicatorVariants({ variant }), className)} />
{showStatus &&
<div className={cn(nodeStatusIndicatorVariants({ variant }), className)} />}
</div>
)
}
@@ -51,16 +51,16 @@ function TransformsPage() {
<div className="max-w-7xl mx-auto p-8">
<div className="flex items-center justify-between">
<div className="space-y-1">
<h1 className="text-3xl font-bold text-foreground">Transforms</h1>
<h1 className="text-3xl font-bold text-foreground">Flows</h1>
<p className="text-muted-foreground">
Create and manage your transforms.
Create and manage your transform flows.
</p>
</div>
<div className="flex items-center gap-2">
<NewTransform>
<Button size="sm">
<PlusIcon className="w-4 h-4 mr-2" />
New Transform
New flow
</Button>
</NewTransform>
</div>
@@ -1,55 +1,43 @@
import { createFileRoute } from '@tanstack/react-router'
import { useState } from 'react'
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'
import { ThirdPartyKeysService } from '../api/third-party-keys-service'
import { KeyService } from '../api/key-service'
import { Button } from '../components/ui/button'
import { Input } from '../components/ui/input'
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '../components/ui/card'
import { Badge } from '../components/ui/badge'
import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, DialogTrigger } from '../components/ui/dialog'
import { Label } from '../components/ui/label'
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '../components/ui/select'
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '../components/ui/table'
import { Loader2, Plus, ExternalLink, Trash2, CheckCircle, XCircle, Clock, Shield } from 'lucide-react'
import { Loader2, Plus, Trash2, Clock, Shield, Key, Zap, Lock, Sparkles } from 'lucide-react'
import { toast } from 'sonner'
import { useConfirm } from '../components/use-confirm-dialog'
import Loader from '@/components/loader'
import { type Key as KeyType } from '@/types/key'
export const Route = createFileRoute('/_auth/dashboard/vault')({
component: VaultPage,
})
interface ServiceInfo {
service: string
variable: string
url: string
active: boolean
}
function VaultPage() {
const [isAddDialogOpen, setIsAddDialogOpen] = useState(false)
const [selectedService, setSelectedService] = useState('')
const [keyName, setKeyName] = useState('')
const [apiKey, setApiKey] = useState('')
const queryClient = useQueryClient()
const { confirm } = useConfirm()
// Fetch services and keys
const { data: services = [], isLoading: servicesLoading } = useQuery({
queryKey: ['third-party-services'],
queryFn: () => ThirdPartyKeysService.getServices(),
})
const { data: keys = [], isLoading: keysLoading } = useQuery({
queryKey: ['third-party-keys'],
queryFn: () => ThirdPartyKeysService.get(),
// Fetch keys
const { data: keys = [], isLoading: keysLoading } = useQuery<KeyType[]>({
queryKey: ['keys'],
queryFn: () => KeyService.get(),
})
// Create key mutation
const createKeyMutation = useMutation({
mutationFn: ThirdPartyKeysService.create,
mutationFn: KeyService.create,
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: ['third-party-keys'] })
queryClient.invalidateQueries({ queryKey: ['keys'] })
setIsAddDialogOpen(false)
setSelectedService('')
setKeyName('')
setApiKey('')
toast.success('API key added successfully!')
},
@@ -61,9 +49,9 @@ function VaultPage() {
// Delete key mutation
const deleteKeyMutation = useMutation({
mutationFn: ThirdPartyKeysService.deleteById,
mutationFn: KeyService.deleteById,
onSuccess: () => {
queryClient.invalidateQueries({ queryKey: ['third-party-keys'] })
queryClient.invalidateQueries({ queryKey: ['keys'] })
toast.success('API key deleted successfully!')
},
onError: (error) => {
@@ -73,17 +61,17 @@ function VaultPage() {
})
const handleAddKey = () => {
if (!selectedService || !apiKey.trim()) {
toast.error('Please select a service and enter an API key')
if (!keyName.trim() || !apiKey.trim()) {
toast.error('Please enter both a name and an API key')
return
}
createKeyMutation.mutate({ service: selectedService, key: apiKey })
createKeyMutation.mutate({ name: keyName.trim(), key: apiKey })
}
const handleDeleteKey = async (keyId: string, serviceName: string) => {
const handleDeleteKey = async (keyId: string, keyName: string) => {
const confirmed = await confirm({
title: 'Delete API Key',
message: `Are you sure you want to delete the API key for ${serviceName}? This action cannot be undone.`
message: `Are you sure you want to delete the API key "${keyName}"? This action cannot be undone.`
})
if (confirmed) {
@@ -91,33 +79,13 @@ function VaultPage() {
}
}
const getKeyForService = (serviceName: string) => {
return keys.find(key => key.service === serviceName)
}
const getStatusIcon = (service: ServiceInfo) => {
const key = getKeyForService(service.service)
if (key) {
return <CheckCircle className="w-5 h-5 text-green-500" />
}
return <XCircle className="w-5 h-5 text-red-500" />
}
const getStatusBadge = (service: ServiceInfo) => {
const key = getKeyForService(service.service)
if (key) {
return <Badge variant="default" className="bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200">Configured</Badge>
}
return <Badge variant="secondary" className="bg-muted text-muted-foreground">Not configured</Badge>
}
if (servicesLoading || keysLoading) {
if (keysLoading) {
return (
<div className="h-full w-full px-12 py-12 bg-background overflow-auto">
<div className='max-w-7xl mx-auto flex h-full flex-col gap-12 items-center justify-start'>
<div className='w-full'>
<h1 className="font-semibold text-2xl">Vault</h1>
<p className="opacity-60 mt-3">Here are the keys used to query third party services.</p>
<p className="opacity-60 mt-3">Manage your API keys for third-party services.</p>
</div>
<div className="w-full h-full flex items-center justify-center">
<Loader />
@@ -130,15 +98,18 @@ function VaultPage() {
return (
<div className="h-full w-full px-12 py-12 bg-background overflow-auto">
<div className='max-w-7xl mx-auto flex flex-col gap-12 items-center justify-start'>
<div className='w-full flex justify-between items-start'>
<div className='w-full flex justify-between items-center'>
<div>
<h1 className="font-semibold text-2xl">Vault</h1>
<p className="opacity-60 mt-3">Here are the keys used to query third party services.</p>
<h1 className="font-semibold text-3xl bg-gradient-to-r from-foreground to-foreground/70 bg-clip-text text-transparent">
Vault
</h1>
<p className="text-muted-foreground mt-2">Securely manage your API keys for third-party services.</p>
</div>
<Dialog open={isAddDialogOpen} onOpenChange={setIsAddDialogOpen}>
<DialogTrigger asChild>
<Button className="flex items-center gap-2">
<Plus className="w-4 h-4" />
<Button
>
<Plus className="w-5 h-5" />
Add API Key
</Button>
</DialogTrigger>
@@ -146,24 +117,18 @@ function VaultPage() {
<DialogHeader>
<DialogTitle>Add API Key</DialogTitle>
<DialogDescription>
Add a new API key for a third-party service. Your keys are encrypted and stored securely.
Add a new API key with a custom name. Your keys are encrypted and stored securely.
</DialogDescription>
</DialogHeader>
<div className="grid gap-4 py-4">
<div className="grid gap-2">
<Label htmlFor="service">Service</Label>
<Select value={selectedService} onValueChange={setSelectedService}>
<SelectTrigger>
<SelectValue placeholder="Select a service" />
</SelectTrigger>
<SelectContent>
{services.filter(s => s.active).map((service) => (
<SelectItem key={service.service} value={service.service}>
{service.service}
</SelectItem>
))}
</SelectContent>
</Select>
<Label htmlFor="keyName">Key Name</Label>
<Input
id="keyName"
placeholder="e.g., OpenAI, GitHub, Shodan..."
value={keyName}
onChange={(e) => setKeyName(e.target.value)}
/>
</div>
<div className="grid gap-2">
<Label htmlFor="apiKey">API Key</Label>
@@ -186,7 +151,7 @@ function VaultPage() {
</Button>
<Button
onClick={handleAddKey}
disabled={createKeyMutation.isPending || !selectedService || !apiKey.trim()}
disabled={createKeyMutation.isPending || !keyName.trim() || !apiKey.trim()}
>
{createKeyMutation.isPending && <Loader2 className="w-4 h-4 mr-2 animate-spin" />}
Add Key
@@ -197,105 +162,134 @@ function VaultPage() {
</div>
<div className="w-full">
<Card>
<CardHeader>
<CardTitle className="flex items-center gap-2">
<Shield className="w-5 h-5" />
Third-Party Services
</CardTitle>
<CardDescription>
Manage your API keys for external services. Configured services will be available for your investigations.
</CardDescription>
</CardHeader>
<CardContent>
<div className="rounded-md border">
{keys.length === 0 ? (
<Card className="border-2 border-dashed border-primary/20 bg-gradient-to-br from-primary/5 via-background to-accent/5">
<CardContent className="flex flex-col items-center justify-center py-20 px-8">
<div className="text-center space-y-4 max-w-md">
<h3 className="text-2xl font-bold bg-gradient-to-r from-foreground to-foreground/70 bg-clip-text text-transparent">
Your Vault is Empty
</h3>
<p className="text-muted-foreground text-lg leading-relaxed">
Add your first API key to unlock the power of third-party services in your investigations.
</p>
<div className="flex flex-wrap justify-center gap-3 pt-4 pb-6">
<Badge variant="secondary" className="bg-primary/10 text-primary border-primary/20 px-4 py-2">
<Zap className="w-4 h-4 mr-2" />
Fast Setup
</Badge>
<Badge variant="secondary" className="bg-emerald-50 text-emerald-700 border-emerald-200 px-4 py-2">
<Lock className="w-4 h-4 mr-2" />
Encrypted
</Badge>
<Badge variant="secondary" className="bg-violet-50 text-violet-700 border-violet-200 px-4 py-2">
<Sparkles className="w-4 h-4 mr-2" />
Secure
</Badge>
</div>
<Button
onClick={() => setIsAddDialogOpen(true)}
>
<Plus className="w-5 h-5 mr-2" />
Add Your First Key
</Button>
</div>
</CardContent>
</Card>
) : (
<Card className="overflow-hidden border bg-gradient-to-br from-background to-muted/20">
<CardHeader className="border-b">
<div className="flex items-center justify-between">
<CardTitle className="flex items-center gap-3 text-xl">
<div className="p-2 bg-primary rounded-lg">
<Shield className="w-5 h-5 text-white" strokeWidth={1.9} />
</div>
API Keys
</CardTitle>
<Badge variant="secondary" className="px-3 py-1">
{keys.length} {keys.length === 1 ? 'key' : 'keys'}
</Badge>
</div>
<CardDescription className="text-base mt-2">
Your encrypted API keys for external services. These keys will be available for your investigations.
</CardDescription>
</CardHeader>
<CardContent className="p-0">
<Table>
<TableHeader>
<TableRow>
<TableHead className="w-12"></TableHead>
<TableHead>Service</TableHead>
<TableHead>Status</TableHead>
<TableHead>Added</TableHead>
<TableHead className="text-right">Actions</TableHead>
<TableRow className="border-b bg-muted/30">
<TableHead className="py-4 px-6 text-sm font-semibold w-2/5">
<div className="flex items-center gap-2">
<Key className="w-4 h-4" />
Name
</div>
</TableHead>
<TableHead className="py-4 text-sm font-semibold w-1/3">
<div className="flex items-center gap-2">
<Clock className="w-4 h-4" />
Created
</div>
</TableHead>
<TableHead className="py-4 px-6 text-sm font-semibold text-right w-1/5">
Actions
</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{services.map((service) => {
const key = getKeyForService(service.service)
return (
<TableRow key={service.service} className="hover:bg-muted/50">
<TableCell className="w-12">
{getStatusIcon(service)}
</TableCell>
<TableCell>
<div className="flex gap-1">
<div className="font-medium">{service.service}</div>
{!service.active && (
<Badge variant="outline" className="w-fit text-orange-600 border-orange-200 dark:text-orange-400 dark:border-orange-800 text-xs">
Coming Soon
</Badge>
)}
{keys.map((key: KeyType) => (
<TableRow
key={key.id}
className="group hover:bg-gradient-to-r hover:from-primary/5 hover:to-accent/5 transition-all duration-200 border-b border-border/50"
>
<TableCell className="py-5 px-6">
<div className="flex items-center gap-4">
<div className="p-2 bg-gradient-to-r from-primary/10 to-accent/10 border border-primary/20 rounded-lg group-hover:scale-110 transition-transform duration-200">
<Key className="w-4 h-4 text-primary" />
</div>
</TableCell>
<TableCell>
{getStatusBadge(service)}
</TableCell>
<TableCell>
{key ? (
<div className="flex items-center gap-2 text-sm text-muted-foreground">
<Clock className="w-4 h-4" />
{new Date(key.created_at).toLocaleDateString()}
<div>
<div className="font-semibold text-foreground group-hover:text-primary transition-colors">
{key.name}
</div>
<div className="text-sm text-muted-foreground">
Encrypted & Secure
</div>
) : (
<span className="text-sm text-muted-foreground"></span>
)}
</TableCell>
<TableCell className="text-right">
<div className="flex items-center justify-end gap-2">
<Button
variant="ghost"
size="sm"
onClick={() => window.open(service.url, '_blank')}
className="flex items-center gap-1 h-8 px-2"
>
<ExternalLink className="w-4 h-4" />
<span className="hidden sm:inline">Visit</span>
</Button>
{key ? (
<Button
variant="ghost"
size="sm"
className="text-destructive hover:text-destructive/80 h-8 px-2"
onClick={() => handleDeleteKey(key.id, service.service)}
disabled={deleteKeyMutation.isPending}
>
<Trash2 className="w-4 h-4" />
</Button>
) : (
<Button
variant="outline"
size="sm"
onClick={() => {
setSelectedService(service.service)
setIsAddDialogOpen(true)
}}
disabled={!service.active}
className="flex items-center gap-1 h-8 px-2"
>
<Plus className="w-4 h-4" />
<span className="hidden sm:inline">Add Key</span>
</Button>
)}
</div>
</TableCell>
</TableRow>
)
})}
</div>
</TableCell>
<TableCell className="py-5">
<div className="flex items-center gap-2 text-sm">
<div className="p-1.5 bg-muted rounded-full">
<Clock className="w-3 h-3 text-muted-foreground" />
</div>
<span className="text-muted-foreground">
{new Date(key.created_at).toLocaleDateString('en-US', {
year: 'numeric',
month: 'short',
day: 'numeric'
})}
</span>
</div>
</TableCell>
<TableCell className="text-right py-5 px-6">
<Button
variant="ghost"
size="icon"
className="text-destructive hover:text-destructive hover:bg-destructive/10"
onClick={() => handleDeleteKey(key.id, key.name)}
disabled={deleteKeyMutation.isPending}
>
<Trash2 className="w-4 h-4" />
</Button>
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</div>
</CardContent>
</Card>
</CardContent>
</Card>
)}
</div>
</div>
</div>
@@ -1,15 +1,22 @@
import {
Link,
Outlet,
createFileRoute,
useNavigate,
} from '@tanstack/react-router'
import { requireAuth } from '@/lib/auth-utils'
import { Button } from '@/components/ui/button'
import { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle } from '@/components/ui/card'
import { AlertTriangle, Home, RefreshCw, ArrowLeft } from 'lucide-react'
import { useState } from 'react'
export const Route = createFileRoute('/_auth')({
beforeLoad: ({ location }) => {
requireAuth(location.href)
},
component: AuthLayout,
errorComponent: ErrorComponent,
})
function AuthLayout() {
@@ -17,3 +24,34 @@ function AuthLayout() {
<Outlet />
)
}
function ErrorComponent() {
return (
<div className="min-h-screen flex items-center justify-center p-4 bg-background">
<Card className="w-full max-w-md">
<CardHeader className="text-center">
<div className="mx-auto mb-4 flex h-12 w-12 items-center justify-center rounded-full bg-destructive/10">
<AlertTriangle className="h-6 w-6 text-destructive" />
</div>
<CardTitle className="text-xl">Something went wrong</CardTitle>
<CardDescription>
We encountered an unexpected error. Please try again or navigate to a safe location.
</CardDescription>
</CardHeader>
<CardFooter className="flex flex-col gap-2">
<Link to="/dashboard">
<Button
className="flex-1"
>
<Home className="mr-2 h-4 w-4" />
Go to Dashboard
</Button>
</Link>
</CardFooter>
</Card>
</div>
)
}
@@ -2,14 +2,14 @@ import { create } from 'zustand';
import { persist } from 'zustand/middleware';
type GraphControlsStore = {
view: 'force' | 'hierarchy' | 'table' | 'map';
view: 'force' | 'hierarchy' | 'table' | 'map' | 'relationships';
zoomToFit: () => void;
zoomIn: () => void;
zoomOut: () => void;
onLayout: (layout: any) => void;
setActions: (actions: Partial<GraphControlsStore>) => void;
refetchGraph: () => void;
setView: (view: 'force' | 'hierarchy' | 'table' | 'map') => void;
setView: (view: 'force' | 'hierarchy' | 'table' | 'map' | 'relationships') => void;
};
export const useGraphControls = create<GraphControlsStore>()(
@@ -12,40 +12,41 @@ import {
applyEdgeChanges
} from "@xyflow/react"
import { toast } from "sonner"
import { type ScannerNodeData } from "@/types/transform"
export interface NodeData {
class_name: string
module: string
key: string
doc?: string | null
computationState?: 'pending' | 'processing' | 'completed' | 'error'
name?: string
category?: string
type?: string
inputs?: { type: string; properties: Array<{ name: string; type: string }> }
outputs?: { type: string; properties: Array<{ name: string; type: string }> }
color?: string
[key: string]: unknown
}
export type NodeData = ScannerNodeData
export type TransformNode = Node<NodeData>
export type TransformEdge = Edge
export interface TransformState {
// Node State
nodes: TransformNode[]
edges: TransformEdge[]
selectedNode: TransformNode | null
// Edge State
edges: TransformEdge[]
// UI State
loading: boolean
openParamsDialog: boolean
// Node Actions
setNodes: (nodes: TransformNode[] | ((prev: TransformNode[]) => TransformNode[])) => void
setEdges: (edges: TransformEdge[] | ((prev: TransformEdge[]) => TransformEdge[])) => void
onNodesChange: OnNodesChange
setSelectedNode: (node: TransformNode | null) => void
deleteNode: (nodeId: string) => void
updateNode: (node: TransformNode) => void
// Edge Actions
setEdges: (edges: TransformEdge[] | ((prev: TransformEdge[]) => TransformEdge[])) => void
onEdgesChange: OnEdgesChange
onConnect: OnConnect
setSelectedNode: (node: TransformNode | null) => void
// UI Actions
setLoading: (loading: boolean) => void
deleteNode: (nodeId: string) => void
setOpenParamsDialog: (openParamsDialog: boolean, node?: TransformNode) => void
}
// ================================
// DEFAULT STYLES & CONFIGURATION
// ================================
const defaultEdgeStyle = { stroke: "#64748b" }
const defaultMarkerEnd: EdgeMarker = {
type: MarkerType.ArrowClosed,
@@ -54,18 +55,49 @@ const defaultMarkerEnd: EdgeMarker = {
color: "#64748b",
}
// ================================
// TRANSFORM STORE IMPLEMENTATION
// ================================
export const useTransformStore = create<TransformState>((set, get) => ({
// ================================
// STATE INITIALIZATION
// ================================
// Node State
nodes: [] as TransformNode[],
edges: [] as TransformEdge[],
selectedNode: null,
// Edge State
edges: [] as TransformEdge[],
// UI State
loading: false,
openParamsDialog: false,
// ================================
// NODE ACTIONS
// ================================
setNodes: (nodes) => set({ nodes: typeof nodes === 'function' ? nodes(get().nodes) : nodes }),
setEdges: (edges) => set({ edges: typeof edges === 'function' ? edges(get().edges) : edges }),
onNodesChange: (changes) => {
set({
nodes: applyNodeChanges(changes, get().nodes) as TransformNode[],
})
},
setSelectedNode: (node) => set({ selectedNode: node }),
deleteNode: (nodeId) => {
const { nodes, edges } = get()
set({
nodes: nodes.filter((node) => node.id !== nodeId),
edges: edges.filter((edge) => edge.source !== nodeId && edge.target !== nodeId),
selectedNode: nodes.find((node) => node.id === nodeId) ? null : get().selectedNode
})
},
updateNode: (node) => {
set({
nodes: get().nodes.map((n) => n.id === node.id ? node : n),
})
},
// ================================
// EDGE ACTIONS
// ================================
setEdges: (edges) => set({ edges: typeof edges === 'function' ? edges(get().edges) : edges }),
onEdgesChange: (changes) => {
set({
edges: applyEdgeChanges(changes, get().edges),
@@ -76,7 +108,6 @@ export const useTransformStore = create<TransformState>((set, get) => ({
toast.error(`Cannot connect ${connection.sourceHandle} to ${connection.targetHandle}.`)
return
}
const edge: TransformEdge = {
id: `${connection.source}-${connection.target}`,
source: connection.source!,
@@ -90,14 +121,19 @@ export const useTransformStore = create<TransformState>((set, get) => ({
edges: [...get().edges, edge],
})
},
setSelectedNode: (node) => set({ selectedNode: node }),
// ================================
// UI ACTIONS
// ================================
setLoading: (loading) => set({ loading }),
deleteNode: (nodeId) => {
const { nodes, edges } = get()
set({
nodes: nodes.filter((node) => node.id !== nodeId),
edges: edges.filter((edge) => edge.source !== nodeId && edge.target !== nodeId),
selectedNode: nodes.find((node) => node.id === nodeId) ? null : get().selectedNode
})
setOpenParamsDialog: (openParamsDialog, node) => {
// Only allow opening the dialog if there's a selected node
if (node) {
set({ selectedNode: node })
}
if (openParamsDialog && !get().selectedNode) {
toast.error("Please select a node first to configure its parameters.")
return
}
set({ openParamsDialog })
},
}))
}))
@@ -0,0 +1,10 @@
export interface Analysis {
id: string; // UUID
title: string;
description?: string | null;
content?: any; // JSONB, so can be any type
created_at: string; // ISO date string
last_updated_at: string; // ISO date string
owner_id?: string | null; // UUID
investigation_id?: string | null; // UUID
}
@@ -0,0 +1,16 @@
export interface ChatMessage {
id: string,
content: string,
is_bot: boolean,
created_at: string,
context?: any,
chatId?: string
}
export interface Chat {
id: string,
title: string,
description: string,
created_at: string,
last_updated_at: string,
}
@@ -0,0 +1,5 @@
import { type SVGProps } from "react";
export type IconSvgProps = SVGProps<SVGSVGElement> & {
size?: number;
};
@@ -0,0 +1,30 @@
import type { GraphEdge, GraphNode } from "@/stores/graph-store";
export enum EventLevel {
// Standard log levels
INFO = "INFO",
WARNING = "WARNING",
FAILED = "FAILED",
SUCCESS = "SUCCESS",
DEBUG = "DEBUG",
// Scanner-specific statuses
PENDING = "PENDING",
RUNNING = "RUNNING",
COMPLETED = "COMPLETED",
GRAPH_APPEND = "GRAPH_APPEND",
}
export interface Payload {
message: string
nodes?: GraphNode[]
edges?: GraphEdge[]
}
export type Event = {
id: string
scan_id: string
sketch_id: string | null
type: EventLevel
payload: Payload
created_at: string
}
@@ -0,0 +1,26 @@
import type { Edge, Node } from "@xyflow/react";
export type NodeData = {
id: string;
type: string,
caption: string,
label: string,
created_at: string,
// Allow any other properties
[key: string]: any;
};
export type EdgeData = {
from: string,
to: string,
date: string,
id: string;
label: string;
type: string,
confidence_level?: number | string
};
export type InvestigationGraph = {
nodes: Node[];
edges: Edge[];
};
+12 -145
View File
@@ -1,145 +1,12 @@
import { GraphEdge, GraphNode } from "@/stores/graph-store";
import type { Edge, Node } from "@xyflow/react";
import { type SVGProps } from "react";
export enum EventLevel {
// Standard log levels
INFO = "INFO",
WARNING = "WARNING",
FAILED = "FAILED",
SUCCESS = "SUCCESS",
DEBUG = "DEBUG",
// Scanner-specific statuses
PENDING = "PENDING",
RUNNING = "RUNNING",
COMPLETED = "COMPLETED",
GRAPH_APPEND = "GRAPH_APPEND",
}
export type IconSvgProps = SVGProps<SVGSVGElement> & {
size?: number;
};
export type NodeData = {
id: string;
type: string,
caption: string,
label: string,
created_at: string,
// Allow any other properties
[key: string]: any;
};
export type EdgeData = {
// source: string;
// target: string;
from: string,
to: string,
date: string,
id: string;
label: string;
type: string,
confidence_level?: number | string
};
export type InvestigationGraph = {
nodes: Node[];
edges: Edge[];
};
export interface Tool {
name: string
path: string
description?: string,
active: boolean
link?: string
avatar?: string
apiKeyRequired?: false | "free" | "paid"
}
export interface ToolCategory {
[key: string]: {
[key: string]: Tool
}
}
export interface Tools {
[key: string]: ToolCategory
}
export interface Profile {
owner?: boolean,
first_name: string,
last_name: string,
id: string,
avatar_url?: string
}
type Scanner = {
id: string;
name: string,
items: Scanner[]
// Add other item properties
};
export interface Transform {
id: string;
name: string;
description?: string;
nodes: NodeData[];
edges: EdgeData[];
created_at?: string;
updated_at?: string;
owner?: Profile;
}
export type NodesData = {
items: Scanner[];
initialEdges?: Edge[];
initialNodes?: Node[];
transform?: Transform;
};
export interface Analysis {
id: string; // UUID
title: string;
description?: string | null;
content?: any; // JSONB, so can be any type
created_at: string; // ISO date string
last_updated_at: string; // ISO date string
owner_id?: string | null; // UUID
investigation_id?: string | null; // UUID
}
export interface Payload {
message: string
nodes?: GraphNode[]
edges?: GraphEdge[]
}
export type Event = {
id: string
scan_id: string
sketch_id: string | null
type: EventLevel
payload: Payload
created_at: string
}
export interface ChatMessage {
id: string,
content: string,
is_bot: boolean,
created_at: string,
context?: any,
chatId?: string
}
export interface Chat {
id: string,
title: string,
description: string,
created_at: string,
last_updated_at: string,
}
export * from "./analysis";
export * from "./chat";
export * from "./common";
export * from "./event";
export * from "./graph";
export * from "./investigation";
export * from "./profile";
export * from "./sketch";
export * from "./table";
export * from "./tool";
export * from "./transform";
export * from "./tabs";
@@ -1,4 +1,5 @@
import { type Sketch } from "./sketch"
import { type Profile } from "./profile"
export interface Investigation {
id: string
@@ -7,7 +8,7 @@ export interface Investigation {
sketches: Sketch[]
created_at: string
last_updated_at: string
owner: any
owner: Profile
owner_id: string,
status: string
}
@@ -0,0 +1,6 @@
export interface Key {
id: string,
name: string,
encrypted_key: string,
created_at: string,
}
@@ -0,0 +1,7 @@
export interface Profile {
owner?: boolean,
first_name: string,
last_name: string,
id: string,
avatar_url?: string
}
+1 -39
View File
@@ -1,4 +1,4 @@
import { type Profile } from "."
import { type Profile } from "./profile"
import { type Investigation } from "./investigation"
export interface Sketch {
@@ -16,42 +16,4 @@ export interface Sketch {
investigation?: Investigation
investigation_id: string
members?: { profile: Profile }[]
}
export interface Individual {
id: string
full_name: string
}
export interface Email {
id: string
email: string
}
export interface Phone {
id: string,
number: string
}
export interface Social {
id: string
profile_url: string
username: string
platform: string
}
export interface IP {
id: string
address: string
}
export interface Address {
id: string
address: string
city: string
country: string
zip: string
}
export interface Relation {
id: string
}
-44
View File
@@ -1,44 +0,0 @@
export interface TabsSlice {
tabs: {
/**
* Array of active tabs.
*/
items: TabInfo[];
/**
* Focused tab id.
*/
selectedTabId: string;
/**
* The index of the selected tab in the items array.
*/
selectedTabIndex: number;
/**
* Initializes the tabs. Must call this before using other methods in here.
*/
initialize: () => Promise<void>;
/**
* Sets the selected tab.
*/
setSelectedTab: (tab: TabInfo) => void;
/**
* Removes a tab
*/
remove: (tab: TabInfo) => void;
/**
* Creates a new tab.
*/
add: () => void;
/**
* Reorder tabs in the order of the given TabInfo array.
*/
reorder: (tabs: TabInfo[]) => void;
};
}
@@ -0,0 +1,19 @@
export interface Tool {
name: string
path: string
description?: string,
active: boolean
link?: string
avatar?: string
apiKeyRequired?: false | "free" | "paid"
}
export interface ToolCategory {
[key: string]: {
[key: string]: Tool
}
}
export interface Tools {
[key: string]: ToolCategory
}
@@ -0,0 +1,106 @@
import type { Edge, Node } from "@xyflow/react";
import type { NodeData, EdgeData } from "./graph";
import type { Profile } from "./profile";
// ================================
// SCANNER TYPE DEFINITIONS
// ================================
export interface ScannerProperty {
name: string
type: string
}
export interface ScannerIO {
type: string
properties: ScannerProperty[]
}
export interface ScannerParamSchemaItem {
name: string
type: string
description: string
default: string
required: boolean
}
export interface Scanner {
class_name: string
category: string
name: string
module: string
doc: string | null
inputs: ScannerIO
outputs: ScannerIO
type: string
requires_key: boolean
params: Record<string, string>
params_schema: ScannerParamSchemaItem[]
settings?: Record<string, string>
}
// ================================
// NODE DATA TYPE FOR TRANSFORM STORE
// ================================
export interface ScannerNodeData extends Scanner, Record<string, unknown> {
color?: string
computationState?: "pending" | "processing" | "completed" | "error"
key: string
}
// ================================
// DATA STRUCTURES
// ================================
export interface ScansData {
[category: string]: Scanner[]
}
export interface ScannerData {
items: ScansData
}
// ================================
// COMPONENT PROPS INTERFACES
// ================================
export interface ScannerItemProps {
scanner: Scanner
category: string
}
export interface ScannerNodeProps {
data: ScannerNodeData
isConnectable?: boolean
selected?: boolean
}
// ================================
// ADDITIONAL TRANSFORM TYPES
// ================================
export type ScannerTree = {
id: string;
name: string,
items: ScannerTree[]
// Add other item properties
};
export interface Transform {
id: string;
name: string;
description?: string;
nodes: NodeData[];
edges: EdgeData[];
created_at?: string;
updated_at?: string;
owner?: Profile;
}
export type NodesData = {
items: ScannerTree[];
initialEdges?: Edge[];
initialNodes?: Node[];
transform?: Transform;
};