diff --git a/docker-compose.yml b/docker-compose.yml index 95003c8..e69c6b2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,14 +9,14 @@ services: - "--api.insecure=true" - "--providers.docker" - "--entrypoints.ws.address=:80" - - "--providers.file.watch=true" - - "--entryPoints.websecure.address=:443" + - "--entrypoints.websecure.address=:443" - "--entrypoints.websecure.http.tls.certresolver=myresolver" - "--providers.file.filename=/etc/traefik/dynamic.yml" + - "--providers.file.watch=true" ports: - "80:80" - "443:443" - - "8080:8080" + - "8080:8080" # Dashboard Traefik restart: unless-stopped volumes: - ./certs:/etc/traefik/certs @@ -24,6 +24,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock networks: - traefik + # PostgreSQL database postgres: image: postgres:15 @@ -34,36 +35,37 @@ services: POSTGRES_PASSWORD: flowsint POSTGRES_DB: flowsint ports: - - "5433:5432" + - "5433:5432" # 5433 sur l'hôte pour éviter conflit local volumes: - pg_data:/var/lib/postgresql/data networks: - traefik + # Frontend Electron/Vite + # web: + # build: + # context: flowsint-app + # dockerfile: Dockerfile + # container_name: flowsint-app + # ports: + # - "3000:3000" + # env_file: + # - .env + # networks: + # - traefik + # labels: + # - "traefik.enable=true" + # - "traefik.http.routers.web.rule=Host(`app.flowsint.localhost`)" + # - "traefik.http.services.web.loadbalancer.server.port=3000" + # - "traefik.http.routers.web.entrypoints=websecure" - # Next.js application - web: - build: - context: flowsint-web - dockerfile: Dockerfile - container_name: flowsint-web - ports: - - "3000:3000" - env_file: - - .env - networks: - - traefik - labels: - - "traefik.enable=true" - - "traefik.http.routers.nextjs.rule=Host(`app.flowsint.localhost`)" - - "traefik.http.services.nextjs.loadbalancer.server.port=3000" - - "traefik.http.routers.nextjs.entrypoints=websecure" - - # FastAPI application with Celery + # FastAPI backend api: build: context: flowsint-api dockerfile: Dockerfile + args: + GITHUB_TOKEN: ${GITHUB_TOKEN} container_name: flowsint-api ports: - "5000:5000" @@ -79,14 +81,17 @@ services: - traefik labels: - "traefik.enable=true" - - "traefik.http.routers.app.rule=Host(`api.flowsint.localhost`)" - - "traefik.http.services.app.loadbalancer.server.port=5000" - - "traefik.http.routers.app.entrypoints=websecure" + - "traefik.http.routers.api.rule=Host(`api.flowsint.localhost`)" + - "traefik.http.services.api.loadbalancer.server.port=5000" + - "traefik.http.routers.api.entrypoints=websecure" + # Celery worker celery: build: context: flowsint-api dockerfile: Dockerfile + args: + GITHUB_TOKEN: ${GITHUB_TOKEN} container_name: celery-worker depends_on: - redis @@ -100,14 +105,16 @@ services: networks: - traefik + # Redis for Celery & cache redis: - image: "redis:alpine" + image: redis:alpine container_name: redis-cache ports: - "6379:6379" networks: - traefik + # Neo4j graph database neo4j: image: neo4j:5 container_name: flowsint-neo4j @@ -134,16 +141,14 @@ services: - "traefik.http.services.neo4j.loadbalancer.server.port=7474" - "traefik.http.routers.neo4j.entrypoints=websecure" - networks: traefik: name: traefik driver: bridge volumes: - db-config: + pg_data: neo4j_data: neo4j_logs: neo4j_import: neo4j_plugins: - pg_data: diff --git a/flowsint-api/Dockerfile b/flowsint-api/Dockerfile index 48c8bfa..f74ce2a 100644 --- a/flowsint-api/Dockerfile +++ b/flowsint-api/Dockerfile @@ -2,9 +2,19 @@ FROM python:3.10 WORKDIR /app +# Install git and configure SSH for private repos +RUN apt-get update && \ + apt-get install -y git openssh-client && \ + rm -rf /var/lib/apt/lists/* + +# Create SSH directory and configure git to use SSH +RUN mkdir -p /root/.ssh && \ + ssh-keyscan github.com >> /root/.ssh/known_hosts + COPY . . -RUN pip install --no-cache-dir -r requirements.txt git+https://github.com/soxoj/maigret ghunt +# Use SSH for git operations during pip install +RUN pip install --no-cache-dir -r requirements.txt RUN apt-get update && \ apt-get install -y git whois dnsutils curl && \ diff --git a/flowsint-api/app/api/routes/analysis.py b/flowsint-api/app/api/routes/analysis.py index a9b3f51..0999d5b 100644 --- a/flowsint-api/app/api/routes/analysis.py +++ b/flowsint-api/app/api/routes/analysis.py @@ -82,4 +82,4 @@ def delete_analysis(analysis_id: UUID, db: Session = Depends(get_db), current_us raise HTTPException(status_code=404, detail="Analysis not found") db.delete(analysis) db.commit() - return None \ No newline at end of file + return None diff --git a/flowsint-api/app/api/routes/sketches.py b/flowsint-api/app/api/routes/sketches.py index 8efa959..a226803 100644 --- a/flowsint-api/app/api/routes/sketches.py +++ b/flowsint-api/app/api/routes/sketches.py @@ -138,12 +138,12 @@ async def get_sketch_nodes( rels = [ { "id": str(record["id"]), - "type": "straight", + "type": "custom", "source": str(record["source"]), "target": str(record["target"]), "data": record["data"], "caption": record["type"], - "label": record["type"].lower(), + "label": record["type"], } for record in rels_result ] diff --git a/flowsint-api/app/api/routes/transforms.py b/flowsint-api/app/api/routes/transforms.py index 0376f1b..f4f3f7c 100644 --- a/flowsint-api/app/api/routes/transforms.py +++ b/flowsint-api/app/api/routes/transforms.py @@ -73,7 +73,8 @@ async def get_material_list(): "category": scanner.get("category"), "name": scanner.get("name"), "module": scanner.get("module"), - "doc": scanner.get("doc"), + "documentation": scanner.get("documentation"), + "description": scanner.get("description"), "inputs": scanner.get("inputs"), "outputs": scanner.get("outputs"), "type": "scanner", diff --git a/flowsint-api/app/api/routes/types.py b/flowsint-api/app/api/routes/types.py index bcc291a..eb0c818 100644 --- a/flowsint-api/app/api/routes/types.py +++ b/flowsint-api/app/api/routes/types.py @@ -188,7 +188,6 @@ async def get_types_list(): def extract_input_schema(model: Type[BaseModel], label_key:str, icon: Optional[str]=None) -> Dict[str, Any]: adapter = TypeAdapter(model) - print(model.__name__) schema = adapter.json_schema() # Use the main schema properties, not the $defs type_name = model.__name__ diff --git a/flowsint-api/app/scanners/base.py b/flowsint-api/app/scanners/base.py index 16b475a..15d7372 100644 --- a/flowsint-api/app/scanners/base.py +++ b/flowsint-api/app/scanners/base.py @@ -185,6 +185,16 @@ class Scanner(ABC): """Primary key on which the scanner operates (e.g. domain, IP, etc.)""" pass + @classmethod + def documentation(cls) -> str: + """ + Return formatted markdown documentation for this scanner. + Override this method to provide custom documentation. + Falls back to cleaned docstring if not overridden. + """ + import inspect + return inspect.cleandoc(cls.__doc__ or "No documentation available.") + @classmethod def input_schema(cls) -> Dict[str, Any]: """ @@ -321,3 +331,42 @@ class Scanner(ABC): Logger.error(self.sketch_id, {"message": f"Scanner {self.name()} errored: '{str(e)}'."}) return [] + def create_node(self, node_type: str, key_prop: str, key_value: str, **properties) -> None: + """Simple helper to create a single Neo4j node.""" + if not self.neo4j_conn: + return + + properties['sketch_id'] = self.sketch_id + properties['label'] = properties.get('label', key_value) + + set_clauses = [f"n.{prop} = ${prop}" for prop in properties.keys()] + params = {key_prop: key_value, **properties} + + query = f""" + MERGE (n:{node_type} {{{key_prop}: ${key_prop}}}) + SET {', '.join(set_clauses)} + """ + self.neo4j_conn.query(query, params) + + def create_relationship(self, from_type: str, from_key: str, from_value: str, + to_type: str, to_key: str, to_value: str, rel_type: str) -> None: + """Simple helper to create a relationship between two nodes.""" + if not self.neo4j_conn: + return + + query = f""" + MATCH (from:{from_type} {{{from_key}: $from_value}}) + MATCH (to:{to_type} {{{to_key}: $to_value}}) + MERGE (from)-[:{rel_type} {{sketch_id: $sketch_id}}]->(to) + """ + + self.neo4j_conn.query(query, { + 'from_value': from_value, + 'to_value': to_value, + 'sketch_id': self.sketch_id + }) + + def log_graph_message(self, message: str) -> None: + """Simple helper to log a graph message.""" + Logger.graph_append(self.sketch_id, {"message": message}) + diff --git a/flowsint-api/app/scanners/crypto/wallet_to_nfts.py b/flowsint-api/app/scanners/crypto/wallet_to_nfts.py index 325318c..1bdc108 100644 --- a/flowsint-api/app/scanners/crypto/wallet_to_nfts.py +++ b/flowsint-api/app/scanners/crypto/wallet_to_nfts.py @@ -37,6 +37,10 @@ class CryptoWalletAddressToNFTs(Scanner): def required_params(cls) -> bool: return True + @classmethod + def icon(cls) -> str | None: + return "cryptowallet" + @classmethod def get_params_schema(cls) -> List[Dict[str, Any]]: """Declare required parameters for this scanner""" @@ -136,55 +140,24 @@ class CryptoWalletAddressToNFTs(Scanner): for nfts in results: for nft in nfts: # Create or update wallet node - wallet_query = """ - MERGE (cryptowallet:cryptowallet {wallet: $wallet_address}) - SET cryptowallet.sketch_id = $sketch_id, - cryptowallet.label = $wallet_address, - cryptowallet.caption = $wallet_address, - cryptowallet.type = "cryptowallet" - """ - self.neo4j_conn.query(wallet_query, { - "wallet_address": nft.wallet.address, - "sketch_id": self.sketch_id - }) + self.create_node('cryptowallet', 'wallet', nft.wallet.address, + caption=nft.wallet.address, type='cryptowallet') - # Create or update NFT node - nft_query = """ - MERGE (nft:nft {contract_address: $contract_address, token_id: $token_id}) - SET nft.collection_name = $collection_name, - nft.metadata_url = $metadata_url, - nft.image_url = $image_url, - nft.name = $name, - nft.sketch_id = $sketch_id, - nft.label = $name, - nft.caption = $name, - nft.type = "nft" - """ - self.neo4j_conn.query(nft_query, { - "contract_address": nft.contract_address, - "token_id": nft.token_id, - "collection_name": nft.collection_name, - "metadata_url": nft.metadata_url, - "image_url": nft.image_url, - "name": nft.name, - "sketch_id": self.sketch_id - }) + # Create or update NFT node + nft_key = f"{nft.contract_address}_{nft.token_id}" + self.create_node('nft', 'nft_id', nft_key, + contract_address=nft.contract_address, + token_id=nft.token_id, + collection_name=nft.collection_name, + metadata_url=nft.metadata_url, + image_url=nft.image_url, + name=nft.name, + caption=nft.name, type='nft') # Create relationship from wallet to NFT - owns_query = """ - MATCH (wallet:wallet {wallet: $wallet_address}) - MATCH (nft:nft {contract_address: $contract_address, token_id: $token_id}) - MERGE (wallet)-[r:OWNS]->(nft) - SET r.sketch_id = $sketch_id - """ - self.neo4j_conn.query(owns_query, { - "wallet_address": nft.wallet.address, - "contract_address": nft.contract_address, - "token_id": nft.token_id, - "sketch_id": self.sketch_id - }) - Logger.graph_append(self.sketch_id, {"message": f"Found NFT for {nft.wallet.address}: {nft.contract_address} - {nft.token_id}"}) - + self.create_relationship('cryptowallet', 'wallet', nft.wallet.address, + 'nft', 'nft_id', nft_key, 'OWNS') + self.log_graph_message(f"Found NFT for {nft.wallet.address}: {nft.contract_address} - {nft.token_id}") return results diff --git a/flowsint-api/app/scanners/crypto/wallet_to_transactions.py b/flowsint-api/app/scanners/crypto/wallet_to_transactions.py index 6beb1eb..83792a5 100644 --- a/flowsint-api/app/scanners/crypto/wallet_to_transactions.py +++ b/flowsint-api/app/scanners/crypto/wallet_to_transactions.py @@ -179,26 +179,13 @@ class CryptoWalletAddressToTransactions(Scanner): for transactions in results: for tx in transactions: - # Create or update both wallet nodes in a single operation - wallets_query = """ - MERGE (source:cryptowallet {wallet: $source_address}) - MERGE (target:cryptowallet {wallet: $target_address}) - SET source.sketch_id = $sketch_id, - source.label = $source_address, - source.caption = $source_address, - source.type = "cryptowallet", - target.sketch_id = $sketch_id, - target.label = $target_address, - target.caption = $target_address, - target.type = "cryptowallet" - """ - self.neo4j_conn.query(wallets_query, { - "source_address": tx.source.address, - "target_address": tx.target.address, - "sketch_id": self.sketch_id - }) + # Create or update both wallet nodes + self.create_node('cryptowallet', 'wallet', tx.source.address, + caption=tx.source.address, type="cryptowallet") + self.create_node('cryptowallet', 'wallet', tx.target.address, + caption=tx.target.address, type="cryptowallet") - # Create transaction as an edge between wallets + # Create transaction as an edge between wallets (keeping complex query for transaction properties) tx_query = """ MATCH (source:cryptowallet {wallet: $source}) MATCH (target:cryptowallet {wallet: $target}) @@ -238,7 +225,9 @@ class CryptoWalletAddressToTransactions(Scanner): "contract_address": tx.contract_address, "sketch_id": self.sketch_id }) - Logger.graph_append(self.sketch_id, {"message": f"Transaction on {datetime.fromtimestamp(int(tx.timestamp)).strftime('%Y-%m-%d %H:%M:%S') if tx.timestamp else 'Unknown time'}: {tx.source.address} -> {tx.target.address}"}) + + timestamp_str = datetime.fromtimestamp(int(tx.timestamp)).strftime('%Y-%m-%d %H:%M:%S') if tx.timestamp else 'Unknown time' + self.log_graph_message(f"Transaction on {timestamp_str}: {tx.source.address} -> {tx.target.address}") return results diff --git a/flowsint-api/app/scanners/domains/resolve.py b/flowsint-api/app/scanners/domains/resolve.py index a5e4ec1..0f75969 100644 --- a/flowsint-api/app/scanners/domains/resolve.py +++ b/flowsint-api/app/scanners/domains/resolve.py @@ -28,6 +28,238 @@ class ResolveScanner(Scanner): def key(cls) -> str: return "domain" + @classmethod + def documentation(cls) -> str: + """Return formatted markdown documentation for the domain resolver scanner.""" + return """ +# Domain Resolver Scanner + +Resolve domain names to their corresponding IP addresses using DNS queries. This scanner performs forward DNS resolution to discover the IP addresses associated with domain names and subdomains. + +## Overview + +The Domain Resolver Scanner takes domain names as input and returns their resolved IP addresses. It automatically handles different input formats and validates domains before attempting resolution. + +## Input/Output Types + +- **Input**: `List[Domain]` - Array of domain objects to resolve +- **Output**: `List[Ip]` - Array of resolved IP addresses + +## Input Format Support + +The scanner accepts multiple input formats and automatically converts them: + +### String Format + +```python +["example.com", "subdomain.example.com"] +``` + +### Dictionary Format + +```python +[ + {"domain": "example.com"}, + {"domain": "subdomain.example.com"} +] +``` + +### Domain Object Format + +```python +[ + Domain(domain="example.com", root=True), + Domain(domain="subdomain.example.com", root=False) +] +``` + +## Resolution Process + +### 1. Input Validation + +- Validates domain format using built-in validation +- Determines if domain is root domain or subdomain +- Filters out invalid domains + +### 2. DNS Resolution + +- Uses Python's `socket.gethostbyname()` for DNS queries +- Resolves each domain to its primary A record +- Handles resolution errors gracefully + +### 3. Result Storage + +- Stores domain-to-IP relationships in Neo4j graph database +- Creates nodes for both domains and IP addresses +- Establishes `RESOLVES_TO` relationships + +## Example Usage + +### Basic Domain Resolution + +**Input:** +```json +[ + "google.com", + "github.com", + "stackoverflow.com" +] +``` + +**Expected Output:** +```json +[ + {"address": "142.250.191.14"}, + {"address": "140.82.113.4"}, + {"address": "151.101.193.69"} +] +``` + +### Mixed Input Types + +**Input:** +```json +[ + "example.com", + {"domain": "subdomain.example.com"}, + {"domain": "api.example.com", "root": false} +] +``` + +## Graph Database Storage + +### Node Creation + +**Domain Node:** +```cypher +MERGE (d:domain {domain: "example.com"}) +SET d.sketch_id = "sketch-uuid", + d.label = "example.com", + d.type = "domain" // or "subdomain" +``` + +**IP Node:** +```cypher +MERGE (ip:ip {address: "93.184.216.34"}) +SET ip.sketch_id = "sketch-uuid", + ip.label = "93.184.216.34", + ip.type = "ip" +``` + +### Relationship Creation + +```cypher +MERGE (d)-[:RESOLVES_TO {sketch_id: "sketch-uuid"}]->(ip) +``` + +## Domain Type Classification + +The scanner automatically classifies domains: + +- **Root Domain**: `example.com` → `type: "domain"` +- **Subdomain**: `api.example.com` → `type: "subdomain"` + +## Error Handling + +### Resolution Failures + +When DNS resolution fails, the scanner: + +- Logs the error with domain name +- Continues processing remaining domains +- Does not create nodes for failed resolutions + +Common resolution failures: + +- **NXDOMAIN**: Domain does not exist +- **Timeout**: DNS server not responding +- **Network errors**: Connectivity issues + +### Invalid Input Handling + +The scanner filters out: + +- Malformed domain names +- Empty strings +- Non-string, non-dict, non-Domain inputs + +## Performance Considerations + +### Resolution Speed + +- Sequential DNS queries (not parallelized) +- Typical resolution time: 10-100ms per domain +- Consider batch size for large domain lists + +### DNS Caching + +- Relies on system DNS cache +- Results may vary based on TTL values +- Fresh queries may take longer than cached ones + +## Use Cases + +### Investigation Scenarios + +1. **Domain Enumeration**: Resolve discovered subdomains to find active hosts +2. **Infrastructure Mapping**: Map domain-to-IP relationships for target organization +3. **CDN Detection**: Identify content delivery network usage patterns +4. **IP Pivoting**: Find shared hosting infrastructure across domains + +### Workflow Integration + +``` +[Domain Discovery] → [Domain Resolver] → [IP Geolocation] + → [Port Scanner] + → [ASN Lookup] +``` + +## Security Considerations + +- **DNS Leakage**: Resolution queries may be logged by DNS providers +- **Rate Limiting**: Some DNS servers may rate limit queries +- **Privacy**: Consider using secure DNS (DoH/DoT) for sensitive investigations + +## Limitations + +- **IPv4 Only**: Currently resolves only A records (IPv4) +- **Single IP**: Returns only the first resolved IP address +- **No CNAME Following**: Does not follow CNAME chains +- **No Cache Control**: Cannot force fresh DNS queries + +## Troubleshooting + +### Common Issues + +1. **No Results**: Check domain validity and DNS configuration +2. **Timeouts**: Verify network connectivity and DNS server availability +3. **Partial Results**: Some domains may fail while others succeed + +### Debug Information + +The scanner provides logging for: +- Input validation results +- DNS resolution attempts +- Graph database operations +- Error conditions + +Check FlowSint logs for detailed resolution information. + +## Technical Details + +### DNS Query Method + +- Uses Python's standard library `socket.gethostbyname()` +- Follows system DNS configuration +- Respects `/etc/hosts` file entries on Unix systems + +### Graph Integration + +- Creates typed nodes in Neo4j +- Maintains investigation context via `sketch_id` +- Supports graph traversal for relationship analysis +""" + def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType: cleaned: InputType = [] for item in data: @@ -55,55 +287,10 @@ class ResolveScanner(Scanner): def postprocess(self, results: OutputType, original_input: InputType) -> OutputType: for domain_obj, ip_obj in zip(original_input, results): - query = """ - MERGE (d:domain {domain: $domain}) - SET d.sketch_id = $sketch_id, - d.label = $domain, - d.type = $type - MERGE (ip:ip {address: $ip}) - SET ip.sketch_id = $sketch_id, - ip.label = $label, - ip.type = "ip" - MERGE (d)-[:RESOLVES_TO {sketch_id: $sketch_id}]->(ip) - """ - if self.neo4j_conn: - self.neo4j_conn.query(query, { - "domain": domain_obj.domain, - "ip": ip_obj.address, - "sketch_id": self.sketch_id, - "label": ip_obj.address, - "type": "domain" if domain_obj.root else "subdomain" - }) - nodes = [Node( - id=str(uuid.uuid4()), - data={ - "label": ip_obj.address, - "type": "ip" - }, - position={ - "x": 0, - "y": 0, - }, - type="ip" - )] - edges =[Edge( - id=str(uuid.uuid4()), - source="", - target="", - data={ - "label": "RESOLVES_TO", - }, - )] - nodes=[node.model_dump_json() for node in nodes] - edges=[edge.model_dump_json() for edge in edges] - payload:Dict = { - "message": f"IP found for domain {domain_obj.domain} -> {ip_obj.address}" - # "nodes": nodes, - # "edges": edges - } - Logger.graph_append(self.sketch_id, payload) - - + self.create_node('domain', 'domain', domain_obj.domain, type='domain' if domain_obj.root else 'subdomain') + self.create_node('ip', 'address', ip_obj.address, type='ip') + self.create_relationship('domain', 'domain', domain_obj.domain, 'ip', 'address', ip_obj.address, 'RESOLVES_TO') + self.log_graph_message(f"IP found for domain {domain_obj.domain} -> {ip_obj.address}") return results # Make types available at module level for easy access diff --git a/flowsint-api/app/scanners/domains/subdomains.py b/flowsint-api/app/scanners/domains/subdomains.py index a64ad77..ade3752 100644 --- a/flowsint-api/app/scanners/domains/subdomains.py +++ b/flowsint-api/app/scanners/domains/subdomains.py @@ -103,21 +103,16 @@ class SubdomainScanner(Scanner): for subdomain in domain_obj["subdomains"]: output.append(Domain(domain=subdomain)) Logger.info(self.sketch_id, {"message": f"{domain_obj['domain']} -> {subdomain}"}) - self.neo4j_conn.query(""" - MERGE (sub:domain {domain: $subdomain}) - SET sub.sketch_id = $sketch_id, - sub.label = $label, - sub.type = $type - MERGE (d:domain {domain: $domain}) - MERGE (d)-[:HAS_SUBDOMAIN {sketch_id: $sketch_id}]->(sub) - """, { - "domain": domain_obj["domain"], - "subdomain": subdomain, - "sketch_id": self.sketch_id, - "label": subdomain, - "type": "subdomain" - }) - Logger.graph_append(self.sketch_id, {"message":f"{domain_obj['domain']} -> {len(domain_obj['subdomains'])} subdomain(s) found."}) + + # Create subdomain node + self.create_node('domain', 'domain', subdomain, + type='domain') + + # Create relationship from parent domain to subdomain + self.create_relationship('domain', 'domain', domain_obj["domain"], + 'domain', 'domain', subdomain, 'HAS_SUBDOMAIN') + + self.log_graph_message(f"{domain_obj['domain']} -> {len(domain_obj['subdomains'])} subdomain(s) found.") return output diff --git a/flowsint-api/app/scanners/domains/to_website.py b/flowsint-api/app/scanners/domains/to_website.py index a23acea..4d4731a 100644 --- a/flowsint-api/app/scanners/domains/to_website.py +++ b/flowsint-api/app/scanners/domains/to_website.py @@ -1,10 +1,9 @@ -from typing import List, Dict, Any, Union +from typing import List, Union import requests -from app.utils import is_valid_domain, resolve_type +from app.utils import is_valid_domain from app.scanners.base import Scanner from app.types.domain import Domain from app.types.website import Website -from pydantic import TypeAdapter from app.core.logger import Logger class DomainToWebsiteScanner(Scanner): @@ -51,7 +50,7 @@ class DomainToWebsiteScanner(Scanner): https_url = f"https://{domain.domain}" response = requests.head(https_url, timeout=10, allow_redirects=True) if response.status_code < 400: - results.append(Website(url=https_url)) + results.append(Website(url=https_url, domain=domain)) continue except requests.RequestException: pass @@ -61,18 +60,18 @@ class DomainToWebsiteScanner(Scanner): http_url = f"http://{domain.domain}" response = requests.head(http_url, timeout=10, allow_redirects=True) if response.status_code < 400: - results.append(Website(url=http_url)) + results.append(Website(url=http_url, domain=domain)) continue except requests.RequestException: pass # If both fail, still add HTTPS URL as default - results.append(Website(url=f"https://{domain.domain}")) + results.append(Website(url=f"https://{domain.domain}", domain=domain)) except Exception as e: Logger.error(self.sketch_id, {"message": f"Error converting domain {domain.domain} to website: {e}"}) # Add HTTPS URL as fallback - results.append(Website(url=f"https://{domain.domain}")) + results.append(Website(url=f"https://{domain.domain}", domain=domain)) return results @@ -87,36 +86,25 @@ class DomainToWebsiteScanner(Scanner): } Logger.info(self.sketch_id, redirect_payload) - query = """ - MERGE (d:domain {domain: $domain}) - SET d.sketch_id = $sketch_id, - d.label = $domain, - d.type = "domain" - MERGE (w:website {url: $url}) - SET w.sketch_id = $sketch_id, - w.label = $label, - w.active = $active, - w.redirects = $redirects, - w.type = "website" - MERGE (d)-[:HAS_WEBSITE {sketch_id: $sketch_id}]->(w) - """ if self.neo4j_conn: - self.neo4j_conn.query(query, { - "domain": website.domain.domain, - "sketch_id": self.sketch_id, - "label": str(website.url), - "active": website.active, - "url": str(website.url), - "redirects": [str(redirect) for redirect in website.redirects] if website.redirects else [], - }) + # Create domain node + self.create_node('domain', 'domain', website.domain.domain, type="domain") + + # Create website node + self.create_node('website', 'url', str(website.url), + active=website.active, + redirects=[str(redirect) for redirect in website.redirects] if website.redirects else [], + type="website") + + # Create relationship + self.create_relationship('domain', 'domain', website.domain.domain, + 'website', 'url', str(website.url), 'HAS_WEBSITE') is_active_str = "active" if website.active else "inactive" redirects_str = f" (redirects: {len(website.redirects)})" if website.redirects else "" - payload:Dict = { - "message": f"{website.domain.domain} -> {str(website.url)} ({is_active_str}){redirects_str}" - } - Logger.graph_append(self.sketch_id, payload) - + self.log_graph_message(f"{website.domain.domain} -> {str(website.url)} ({is_active_str}){redirects_str}") + + return results # Make types available at module level for easy access diff --git a/flowsint-api/app/scanners/domains/whois.py b/flowsint-api/app/scanners/domains/whois.py index 007fc3c..233e7ba 100644 --- a/flowsint-api/app/scanners/domains/whois.py +++ b/flowsint-api/app/scanners/domains/whois.py @@ -1,12 +1,11 @@ import json -from typing import List, Dict, Any, Union +from typing import List, Union import whois -from app.utils import is_valid_domain, resolve_type +from app.utils import is_valid_domain from app.scanners.base import Scanner from app.types.domain import Domain, Domain from app.types.whois import Whois from app.types.email import Email -from pydantic import TypeAdapter from app.core.logger import Logger class WhoisScanner(Scanner): @@ -58,14 +57,30 @@ class WhoisScanner(Scanner): else: emails = [Email(email=whois_info.emails)] + # Convert datetime objects to ISO format strings + creation_date_str = None + if whois_info.creation_date: + if isinstance(whois_info.creation_date, list): + creation_date_str = whois_info.creation_date[0].isoformat() if whois_info.creation_date else None + else: + creation_date_str = whois_info.creation_date.isoformat() + + expiration_date_str = None + if whois_info.expiration_date: + if isinstance(whois_info.expiration_date, list): + expiration_date_str = whois_info.expiration_date[0].isoformat() if whois_info.expiration_date else None + else: + expiration_date_str = whois_info.expiration_date.isoformat() + whois_obj = Whois( domain=domain.domain, registrar=str(whois_info.registrar) if whois_info.registrar else None, - creation_date=whois_info.creation_date, - expiration_date=whois_info.expiration_date, - name_servers=whois_info.name_servers if whois_info.name_servers else [], - emails=emails, - raw_text=str(whois_info) + org=str(whois_info.org) if whois_info.org else None, + city=str(whois_info.city) if whois_info.city else None, + country=str(whois_info.country) if whois_info.country else None, + email=emails[0] if emails else None, + creation_date=creation_date_str, + expiration_date=expiration_date_str ) results.append(whois_obj) @@ -76,92 +91,50 @@ class WhoisScanner(Scanner): return results def postprocess(self, results: OutputType, original_input: InputType) -> OutputType: - for domain in results: + for whois_obj in results: if not self.neo4j_conn: continue - whois_obj = domain["whois"] - Logger.graph_append(self.sketch_id, {"message": f"WHOIS for {domain['domain']} -> registrar: {whois_obj.registrar} org: {whois_obj.org} city: {whois_obj.city} country: {whois_obj.country} creation_date: {whois_obj.creation_date} expiration_date: {whois_obj.expiration_date}"}) - props = { - "domain": domain["domain"], - "registrar": whois_obj.registrar, - "org": whois_obj.org, - "city": whois_obj.city, - "country": whois_obj.country, - "creation_date": whois_obj.creation_date, - "expiration_date": whois_obj.expiration_date, - "email": whois_obj.email.email if whois_obj.email else None, - "sketch_id": self.sketch_id - } - - query = """ - MERGE (d:domain {domain: $domain}) - SET d.sketch_id = $sketch_id, - d.label = $domain, - d.type = "domain" - MERGE (w:whois {domain: $domain, sketch_id: $sketch_id}) - SET w.registrar = $registrar, - w.org = $org, - w.type = "whois", - w.label = "Whois", - w.city = $city, - w.country = $country, - w.creation_date = $creation_date, - w.expiration_date = $expiration_date, - w.email = $email - MERGE (d)-[:HAS_WHOIS {sketch_id: $sketch_id}]->(w) - """ - self.neo4j_conn.query(query, props) + + # Create domain node + self.create_node('domain', 'domain', whois_obj.domain, type='domain') + + # Create whois node + whois_key = f"{whois_obj.domain}_{self.sketch_id}" + self.create_node('whois', 'whois_id', whois_key, + domain=whois_obj.domain, + registrar=whois_obj.registrar, + org=whois_obj.org, + city=whois_obj.city, + country=whois_obj.country, + creation_date=whois_obj.creation_date, + expiration_date=whois_obj.expiration_date, + email=whois_obj.email.email if whois_obj.email else None, + label="Whois", type="whois") + + # Create relationship between domain and whois + self.create_relationship('domain', 'domain', whois_obj.domain, + 'whois', 'whois_id', whois_key, 'HAS_WHOIS') # Create organization node if org information is available if whois_obj.org: - org_query = """ - MERGE (o:organization {name: $org_name}) - SET o.country = $country, - o.founding_date = $creation_date, - o.description = $description, - o.label = $label, - o.caption = $caption, - o.type = $type, - o.sketch_id = $sketch_id - """ - self.neo4j_conn.query(org_query, { - "org_name": whois_obj.org, - "country": whois_obj.country, - "creation_date": whois_obj.creation_date, - "description": f"Organization from WHOIS data for {domain['domain']}", - "label": whois_obj.org, - "caption": whois_obj.org, - "type": "organization", - "sketch_id": self.sketch_id, - }) + self.create_node('organization', 'name', whois_obj.org, + country=whois_obj.country, + founding_date=whois_obj.creation_date, + description=f"Organization from WHOIS data for {whois_obj.domain}", + caption=whois_obj.org, type="organization") - # Create relationship between domain and organization - self.neo4j_conn.query(""" - MERGE (d:domain {domain: $domain}) - MERGE (o:organization {name: $org_name}) - MERGE (o)-[:HAS_DOMAIN {sketch_id: $sketch_id}]->(d) - """, { - "domain": domain["domain"], - "org_name": whois_obj.org, - "sketch_id": self.sketch_id, - }) + # Create relationship between organization and domain + self.create_relationship('organization', 'name', whois_obj.org, + 'domain', 'domain', whois_obj.domain, 'HAS_DOMAIN') - Logger.graph_append(self.sketch_id, {"message": f"{domain['domain']} -> {whois_obj.org} (organization)"}) + self.log_graph_message(f"{whois_obj.domain} -> {whois_obj.org} (organization)") if whois_obj.email: - email_query = """ - MERGE (e:email {email: $email}) - SET e.sketch_id = $sketch_id, - e.type = "email", - e.label = $email - MERGE (w:whois {domain: $domain, sketch_id: $sketch_id}) - MERGE (w)-[:REGISTERED_BY {sketch_id: $sketch_id}]->(e) - """ - self.neo4j_conn.query(email_query, { - "email": whois_obj.email.email, - "domain": domain["domain"], - "sketch_id": self.sketch_id - }) + self.create_node('email', 'email', whois_obj.email.email, type="email") + self.create_relationship('whois', 'whois_id', whois_key, + 'email', 'email', whois_obj.email.email, 'REGISTERED_BY') + + self.log_graph_message(f"WHOIS for {whois_obj.domain} -> registrar: {whois_obj.registrar} org: {whois_obj.org} city: {whois_obj.city} country: {whois_obj.country} creation_date: {whois_obj.creation_date} expiration_date: {whois_obj.expiration_date}") return results diff --git a/flowsint-api/app/scanners/emails/to_gravatar.py b/flowsint-api/app/scanners/emails/to_gravatar.py index e043a71..99997f6 100644 --- a/flowsint-api/app/scanners/emails/to_gravatar.py +++ b/flowsint-api/app/scanners/emails/to_gravatar.py @@ -56,10 +56,10 @@ class EmailToGravatarScanner(Scanner): profile_response = requests.get(profile_url, timeout=10) gravatar_data = { - "email": email.email, + "src": gravatar_url, "hash": email_hash, - "avatar_url": gravatar_url, - "profile_url": profile_url + "profile_url": profile_url, + "exists": True } if profile_response.status_code == 200: @@ -69,7 +69,7 @@ class EmailToGravatarScanner(Scanner): gravatar_data.update({ "display_name": entry.get("displayName"), "about_me": entry.get("aboutMe"), - "current_location": entry.get("currentLocation") + "location": entry.get("currentLocation") }) gravatar = Gravatar(**gravatar_data) @@ -81,7 +81,32 @@ class EmailToGravatarScanner(Scanner): return results - def postprocess(self, results: OutputType, input_data: InputType = None) -> OutputType: + def postprocess(self, results: OutputType, original_input: InputType) -> OutputType: + for email_obj, gravatar_obj in zip(original_input, results): + if not self.neo4j_conn: + continue + + # Create email node + self.create_node('email', 'email', email_obj.email, type='email') + + # Create gravatar node + gravatar_key = f"{email_obj.email}_{self.sketch_id}" + self.create_node('gravatar', 'gravatar_id', gravatar_key, + email=email_obj.email, + hash=gravatar_obj.hash, + src=str(gravatar_obj.src), + display_name=gravatar_obj.display_name, + location=gravatar_obj.location, + about_me=gravatar_obj.about_me, + exists=gravatar_obj.exists, + label="Gravatar", type="gravatar") + + # Create relationship between email and gravatar + self.create_relationship('email', 'email', email_obj.email, + 'gravatar', 'gravatar_id', gravatar_key, 'HAS_GRAVATAR') + + self.log_graph_message(f"Gravatar found for email {email_obj.email} -> hash: {gravatar_obj.hash}") + return results # Make types available at module level for easy access diff --git a/flowsint-api/app/scanners/emails/to_leaks.py b/flowsint-api/app/scanners/emails/to_leaks.py index 24ff06a..cd58fbf 100644 --- a/flowsint-api/app/scanners/emails/to_leaks.py +++ b/flowsint-api/app/scanners/emails/to_leaks.py @@ -1,5 +1,5 @@ import os -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Optional, Union import requests from urllib.parse import urljoin from app.scanners.base import Scanner @@ -7,6 +7,7 @@ from app.core.logger import Logger from app.types.email import Email from app.types.breach import Breach from dotenv import load_dotenv +from app.core.graph_db import Neo4jConnection # Load environment variables load_dotenv() @@ -17,7 +18,24 @@ class EmailToBreachesScanner(Scanner): """From email to breaches using Have I Been Pwned API.""" InputType = List[Email] - OutputType = List[Breach] + OutputType = List[tuple] # List of (email, breach) tuples + + def __init__( + self, + sketch_id: Optional[str] = None, + scan_id: Optional[str] = None, + neo4j_conn: Optional[Neo4jConnection] = None, + vault=None, + params: Optional[Dict[str, Any]] = None + ): + super().__init__( + sketch_id=sketch_id, + scan_id=scan_id, + neo4j_conn=neo4j_conn, + params_schema=self.get_params_schema(), + vault=vault, + params=params + ) @classmethod def name(cls) -> str: @@ -70,8 +88,9 @@ class EmailToBreachesScanner(Scanner): async def scan(self, data: InputType) -> OutputType: results: OutputType = [] - api_key = self.resolve_params()["HIBP_API_KEY"] - api_url = self.resolve_params()["HIBP_API_URL"] + print(self.get_params()) + api_key = self.get_params().get("HIBP_API_KEY", None) + api_url = self.get_params().get("HIBP_API_URL", None) if not api_key: Logger.error(self.sketch_id, {"message": "A valid HIBP_API_KEY is required to scan for breaches."}) if not api_url: @@ -80,35 +99,38 @@ class EmailToBreachesScanner(Scanner): "hibp-api-key": api_key, "User-Agent": "FlowsInt-Scanner" } - Logger.info(self.sketch_id, {"message": f"HIBP API key: {api_key}"}) Logger.info(self.sketch_id, {"message": f"HIBP API URL: {api_url}"}) for email in data: try: # Query Have I Been Pwned API - full_url = urljoin(api_url, email.email) + full_url = urljoin(api_url, f"{email.email}?truncateResponse=false") + Logger.error(self.sketch_id, {"message": f"full url: {full_url}"}) response = requests.get(full_url, headers=headers, timeout=10) Logger.info(self.sketch_id, {"message": f"HIBP API response: {response.json()}"}) if response.status_code == 200: breaches_data = response.json() + Logger.info(self.sketch_id, {"message": f"Found {len(breaches_data)} breaches for {email.email}"}) for breach_data in breaches_data: breach = Breach( name=breach_data.get("Name", ""), title=breach_data.get("Title", ""), domain=breach_data.get("Domain", ""), - breach_date=breach_data.get("BreachDate", ""), - added_date=breach_data.get("AddedDate", ""), - modified_date=breach_data.get("ModifiedDate", ""), - pwn_count=breach_data.get("PwnCount", 0), + breachdate=breach_data.get("BreachDate", ""), + addeddate=breach_data.get("AddedDate", ""), + modifieddate=breach_data.get("ModifiedDate", ""), + pwncount=breach_data.get("PwnCount", 0), description=breach_data.get("Description", ""), - data_classes=breach_data.get("DataClasses", []), - is_verified=breach_data.get("IsVerified", False), - is_fabricated=breach_data.get("IsFabricated", False), - is_sensitive=breach_data.get("IsSensitive", False), - is_retired=breach_data.get("IsRetired", False), - is_spam_list=breach_data.get("IsSpamList", False), - logo_path=breach_data.get("LogoPath", "") + dataclasses=breach_data.get("DataClasses", []), + isverified=breach_data.get("IsVerified", False), + isfabricated=breach_data.get("IsFabricated", False), + issensitive=breach_data.get("IsSensitive", False), + isretired=breach_data.get("IsRetired", False), + isspamlist=breach_data.get("IsSpamList", False), + logopath=breach_data.get("LogoPath", "") ) - results.append(breach) + # Store email and breach as a tuple + results.append((email.email, breach)) + Logger.info(self.sketch_id, {"message": f"Added breach: {breach.name} for email: {email.email}"}) elif response.status_code == 404: # No breaches found for this email @@ -123,9 +145,39 @@ class EmailToBreachesScanner(Scanner): Logger.error(self.sketch_id, {"message": f"Error checking breaches for email {email.email}: {e}"}) continue + Logger.info(self.sketch_id, {"message": f"Scan completed. Total results: {len(results)}"}) return results - def postprocess(self, results: OutputType, input_data: InputType = None) -> OutputType: + def postprocess(self, results: OutputType, original_input: InputType) -> OutputType: + Logger.info(self.sketch_id, {"message": f"Postprocess started. Results count: {len(results)}, Original input count: {len(original_input)}"}) + + # Create email nodes first + for email_obj in original_input: + if not self.neo4j_conn: + continue + # Create email node + self.create_node('email', 'email', email_obj.email, type='email') + Logger.info(self.sketch_id, {"message": f"Created email node: {email_obj.email}"}) + + # Process all breaches + for email_address, breach_obj in results: + if not self.neo4j_conn: + continue + + # Create breach node + breach_key = f"{breach_obj.name}_{self.sketch_id}" + self.create_node( + 'breach', 'breach_id', breach_key, + **breach_obj.dict(), + label=breach_obj.name, type="breach" + ) + Logger.info(self.sketch_id, {"message": f"Created breach node: {breach_key}"}) + + # Create relationship between the specific email and this breach + self.create_relationship('email', 'email', email_address, + 'breach', 'breach_id', breach_key, 'FOUND_IN_BREACH') + self.log_graph_message(f"Breach found for email {email_address} -> {breach_obj.name} ({breach_obj.title})") + return results # Make types available at module level for easy access diff --git a/flowsint-api/app/scanners/individuals/to_org.py b/flowsint-api/app/scanners/individuals/to_org.py index 12938af..c219afa 100644 --- a/flowsint-api/app/scanners/individuals/to_org.py +++ b/flowsint-api/app/scanners/individuals/to_org.py @@ -227,29 +227,18 @@ class IndividualToOrgScanner(Scanner): for org in results: # Create or update the organization node with all SIRENE properties - self.neo4j_conn.query(""" - MERGE (o:Organization {name: $name, country: $country}) - SET o.siren = $siren, - o.siege_siret = $siret, - o.nom_complet = $nom_complet, - o.nom_raison_sociale = $nom_raison_sociale, - o.sigle = $sigle, - o.sketch_id = $sketch_id, - o.label = $name, - o.caption = $name, - o.type = 'organization' - """, { - "name": org.name, - "country": "FR", - "siren": org.siren, - "siret": org.siege_siret, - "nom_complet": org.nom_complet, - "nom_raison_sociale": org.nom_raison_sociale, - "sigle": org.sigle, - "sketch_id": self.sketch_id, - }) + org_key = f"{org.name}_FR" + self.create_node('Organization', 'org_id', org_key, + name=org.name, + country="FR", + siren=org.siren, + siege_siret=org.siege_siret, + nom_complet=org.nom_complet, + nom_raison_sociale=org.nom_raison_sociale, + sigle=org.sigle, + caption=org.name, type='organization') - Logger.graph_append(self.sketch_id, {"message": f"Found organization: {org.name}"}) + self.log_graph_message(f"Found organization: {org.name}") return results diff --git a/flowsint-api/app/scanners/ips/asn_to_cidrs.py b/flowsint-api/app/scanners/ips/asn_to_cidrs.py index 33cf2d2..0cbb79c 100644 --- a/flowsint-api/app/scanners/ips/asn_to_cidrs.py +++ b/flowsint-api/app/scanners/ips/asn_to_cidrs.py @@ -23,6 +23,10 @@ class AsnToCidrsScanner(Scanner): @classmethod def category(cls) -> str: return "Asn" + + @classmethod + def key(cls) -> str: + return 'network' def preprocess(self, data: Union[List[str], List[int], List[dict], InputType]) -> InputType: cleaned: InputType = [] @@ -120,65 +124,37 @@ class AsnToCidrsScanner(Scanner): if str(cidr.network) == "0.0.0.0/0": continue # Skip default CIDR for unknown ASN if self.neo4j_conn: - query = """ - MERGE (asn:asn {number: $asn_number}) - SET asn.sketch_id = $sketch_id, - asn.name = $asn_name, - asn.country = $asn_country, - asn.label = $asn_label, - asn.caption = $asn_caption, - asn.type = "asn" + self.create_node('asn', 'number', asn.number, + name=asn.name or "Unknown", + country=asn.country or "Unknown", + label=f"AS{asn.number}", + caption=f"AS{asn.number} - {asn.name or 'Unknown'}", + type='asn') - MERGE (cidr:cidr {network: $cidr_network}) - SET cidr.sketch_id = $sketch_id, - cidr.label = $cidr_network, - cidr.caption = $cidr_network, - cidr.type = "cidr" + self.create_node('cidr', 'network', str(cidr.network), + caption=str(cidr.network), type='cidr') - MERGE (asn)-[:ANNOUNCES {sketch_id: $sketch_id}]->(cidr) - """ - self.neo4j_conn.query(query, { - "asn_number": asn.number, - "asn_name": asn.name or "Unknown", - "asn_country": asn.country or "Unknown", - "asn_label": f"AS{asn.number}", - "asn_caption": f"AS{asn.number} - {asn.name or 'Unknown'}", - "cidr_network": str(cidr.network), - "sketch_id": self.sketch_id, - }) + self.create_relationship('asn', 'number', asn.number, + 'cidr', 'network', str(cidr.network), 'ANNOUNCES') else: # Fallback: original behavior (one-to-one zip) for asn, cidr in zip(original_input, results): if str(cidr.network) == "0.0.0.0/0": continue # Skip default CIDR for unknown ASN - Logger.graph_append(self.sketch_id, {"message": f"ASN {asn.number} -> {cidr.network}"}) + self.log_graph_message(f"ASN {asn.number} -> {cidr.network}") if self.neo4j_conn: - query = """ - MERGE (asn:ASN {number: $asn_number}) - SET asn.sketch_id = $sketch_id, - asn.name = $asn_name, - asn.country = $asn_country, - asn.label = $asn_label, - asn.caption = $asn_caption, - asn.type = "asn" + self.create_node('ASN', 'number', asn.number, + name=asn.name or "Unknown", + country=asn.country or "Unknown", + label=f"AS{asn.number}", + caption=f"AS{asn.number} - {asn.name or 'Unknown'}", + type='asn') - MERGE (cidr:CIDR {network: $cidr_network}) - SET cidr.sketch_id = $sketch_id, - cidr.label = $cidr_network, - cidr.caption = $cidr_network, - cidr.type = "cidr" + self.create_node('CIDR', 'network', str(cidr.network), + caption=str(cidr.network), type='cidr') - MERGE (asn)-[:ANNOUNCES {sketch_id: $sketch_id}]->(cidr) - """ - self.neo4j_conn.query(query, { - "asn_number": asn.number, - "asn_name": asn.name or "Unknown", - "asn_country": asn.country or "Unknown", - "asn_label": f"AS{asn.number}", - "asn_caption": f"AS{asn.number} - {asn.name or 'Unknown'}", - "cidr_network": str(cidr.network), - "sketch_id": self.sketch_id, - }) + self.create_relationship('ASN', 'number', asn.number, + 'CIDR', 'network', str(cidr.network), 'ANNOUNCES') return results # Make types available at module level for easy access diff --git a/flowsint-api/app/scanners/ips/cidr_to_ips.py b/flowsint-api/app/scanners/ips/cidr_to_ips.py index d77a4ec..29199d6 100644 --- a/flowsint-api/app/scanners/ips/cidr_to_ips.py +++ b/flowsint-api/app/scanners/ips/cidr_to_ips.py @@ -79,28 +79,20 @@ class CidrToIpsScanner(Scanner): def postprocess(self, results: OutputType, original_input: InputType) -> OutputType: # Create Neo4j relationships between CIDRs and their corresponding IPs for cidr, ip in zip(original_input, results): - Logger.graph_append(self.sketch_id, {"message": f"Found {len(results)} IPs for CIDR {cidr.network}"}) if self.neo4j_conn: - query = """ - MERGE (cidr:cidr {network: $cidr_network}) - SET cidr.sketch_id = $sketch_id, - cidr.label = $cidr_network, - cidr.caption = $cidr_network, - cidr.type = "cidr" + # Create CIDR node + self.create_node('cidr', 'network', str(cidr.network), + caption=str(cidr.network), type="cidr") - MERGE (ip:ip {address: $ip_address}) - SET ip.sketch_id = $sketch_id, - ip.label = $ip_address, - ip.caption = $ip_address, - ip.type = "ip" + # Create IP node + self.create_node('ip', 'address', ip.address, + caption=ip.address, type="ip") - MERGE (cidr)-[:CONTAINS {sketch_id: $sketch_id}]->(ip) - """ - self.neo4j_conn.query(query, { - "cidr_network": str(cidr.network), - "ip_address": ip.address, - "sketch_id": self.sketch_id, - }) + # Create relationship + self.create_relationship('cidr', 'network', str(cidr.network), + 'ip', 'address', ip.address, 'CONTAINS') + + self.log_graph_message(f"Found {len(results)} IPs for CIDR {cidr.network}") return results # Make types available at module level for easy access diff --git a/flowsint-api/app/scanners/ips/geolocation.py b/flowsint-api/app/scanners/ips/geolocation.py index de335e5..7dac10e 100644 --- a/flowsint-api/app/scanners/ips/geolocation.py +++ b/flowsint-api/app/scanners/ips/geolocation.py @@ -91,25 +91,10 @@ class GeolocationScanner(Scanner): """Update IP nodes in Neo4j with geolocation information.""" if self.neo4j_conn: for ip in results: - Logger.graph_append(self.sketch_id, {"message": f"Geolocated {ip.address} to {ip.city}, {ip.country} (lat: {ip.latitude}, lon: {ip.longitude})"}) - query = """ - MATCH (ip:ip {address: $ip_address}) - SET ip.latitude = $latitude, - ip.longitude = $longitude, - ip.country = $country, - ip.city = $city, - ip.isp = $isp, - ip.sketch_id = $sketch_id - """ - self.neo4j_conn.query(query, { - "ip_address": ip.address, - "latitude": ip.latitude, - "longitude": ip.longitude, - "country": ip.country, - "city": ip.city, - "isp": ip.isp, - "sketch_id": self.sketch_id - }) + self.create_node('ip', 'address', ip.address, + latitude=ip.latitude, longitude=ip.longitude, + country=ip.country, city=ip.city, isp=ip.isp, type='ip') + self.log_graph_message(f"Geolocated {ip.address} to {ip.city}, {ip.country} (lat: {ip.latitude}, lon: {ip.longitude})") return results def get_location_data(self, address: str) -> Dict[str, Any]: diff --git a/flowsint-api/app/scanners/leaks/hibp.py b/flowsint-api/app/scanners/leaks/hibp.py index 90428fd..8147458 100644 --- a/flowsint-api/app/scanners/leaks/hibp.py +++ b/flowsint-api/app/scanners/leaks/hibp.py @@ -82,37 +82,18 @@ class HibpScanner(Scanner): email = result["email"] # Create email node - email_query = """ - MERGE (email:email {address: $address}) - SET email.sketch_id = $sketch_id, - email.label = $address, - email.caption = $address, - email.type = "email" - """ - self.neo4j_conn.query(email_query, { - "address": email, - "sketch_id": self.sketch_id - }) + self.create_node('email', 'address', email, + caption=email, type='email') # Create breach relationships for breach in result.get("breaches", []): if breach and isinstance(breach, dict): breach_name = breach.get("Name", "Unknown") - self.neo4j_conn.query(""" - MERGE (breach:breach {name: $name}) - SET breach.sketch_id = $sketch_id, - breach.label = $name, - breach.caption = $name, - breach.type = "breach" - WITH breach - MATCH (email:email {address: $email_address}) - MERGE (email)-[:FOUND_IN_BREACH {sketch_id: $sketch_id}]->(breach) - """, { - "name": breach_name, - "email_address": email, - "sketch_id": self.sketch_id - }) - Logger.graph_append(self.sketch_id, {"message": f"Email {email} found in breach: {breach_name}"}) + self.create_node('breach', 'name', breach_name, + caption=breach_name, type='breach') + self.create_relationship('email', 'address', email, + 'breach', 'name', breach_name, 'FOUND_IN_BREACH') + self.log_graph_message(f"Email {email} found in breach: {breach_name}") return results diff --git a/flowsint-api/app/scanners/n8n/connector.py b/flowsint-api/app/scanners/n8n/connector.py index 6851204..cfddd38 100644 --- a/flowsint-api/app/scanners/n8n/connector.py +++ b/flowsint-api/app/scanners/n8n/connector.py @@ -8,17 +8,6 @@ from app.core.graph_db import Neo4jConnection class N8nConnector(Scanner): """ Connect to your custom n8n workflows to process data through webhooks. - - ## Setup instructions: - 1. In your n8n workflow, add a **Webhook** trigger node as the starting node - 2. In the Webhook node, set **Respond** to `"Using 'Respond to Webhook' node"` - 3. Add a **Respond to Webhook** node at the end of your workflow to return processed data - 4. Use the webhook URL from your n8n workflow in the `webhook_url` parameter - - The connector will send your input data as JSON to the webhook and expect JSON response. - Types are not validated by this connector, so ensure your n8n workflow handles the expected data types correctly. - - For more details on webhook responses, see: [Respond to Webhook documentation](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.respondtowebhook/) """ # Define types as class attributes - base class handles schema generation automatically @@ -42,6 +31,187 @@ class N8nConnector(Scanner): params=params ) + @classmethod + def documentation(cls) -> str: + """Return formatted markdown documentation for the n8n connector.""" + return """ +# n8n Connector + +Connect to your custom n8n workflows to process data through webhooks. This connector allows you to leverage n8n's powerful automation capabilities within your FlowSint investigations. + +## Setup Instructions + +### 1. Configure n8n Workflow + +Create a new workflow in n8n with the following structure: + +``` +[Webhook Trigger] → [Your Processing Nodes] → [Respond to Webhook] +``` + +### 2. Configure Webhook Trigger Node + +In your n8n workflow, add a **Webhook** trigger node as the starting node: + +```json +{ + "httpMethod": "POST", + "path": "your-webhook-path", + "responseMode": "responseNode" +} +``` + +**Important**: Set **Respond** to `"Using 'Respond to Webhook' node"` + +### 3. Add Processing Logic + +Add your custom processing nodes between the webhook trigger and response node. Your workflow will receive the following JSON payload: + +```json +{ + "sketch_id": "sketch-uuid", + "type": "input-data-type", + "inputs": [ + // Your input data array + ] +} +``` + +### 4. Configure Respond to Webhook Node + +Add a **Respond to Webhook** node at the end of your workflow to return processed data: + +```json +{ + "respondWith": "json", + "responseBody": "{{ $json.processedData }}" +} +``` + +## Configuration Parameters + +### Required Parameters + +- **webhook_url** (url): The n8n webhook URL to send data to + ``` + Example: https://your-n8n-instance.com/webhook/your-webhook-id + ``` + +### Optional Parameters + +- **auth_token** (vaultSecret): Authentication token for the webhook + ``` + Used as: Authorization: Bearer + ``` + +- **extra_payload** (string): Additional JSON data to include in the payload + ```json + { + "custom_field": "value", + "metadata": { + "source": "flowsint" + } + } + ``` + +## Input/Output Types + +- **Input**: `List[Any]` - Array of data to be processed +- **Output**: `List[Any]` - Array of processed results from n8n + +## Example Usage + +### Basic Setup + +```json +{ + "webhook_url": "https://n8n.example.com/webhook/abc123" +} +``` + +### With Authentication + +```json +{ + "webhook_url": "https://n8n.example.com/webhook/abc123", + "auth_token": "your-secret-token" +} +``` + +### With Extra Payload + +```json +{ + "webhook_url": "https://n8n.example.com/webhook/abc123", + "extra_payload": "{\"priority\": \"high\", \"source\": \"investigation\"}" +} +``` + +## Sample n8n Workflow Response + +Your n8n workflow should return data in this format: + +```json +[ + { + "id": "processed-item-1", + "result": "processed data", + "metadata": { + "processed_at": "2024-01-01T12:00:00Z" + } + } +] +``` + +## Error Handling + +The connector handles various error scenarios: + +- **Non-200 HTTP responses**: Logs error and raises exception +- **Invalid JSON responses**: Returns error object with raw response +- **Network errors**: Logs error and re-raises exception + +Example error response: +```json +[ + { + "raw_response": "Server Error", + "error": "Response was not valid JSON" + } +] +``` + +## Security Considerations + +- Use HTTPS URLs for webhook endpoints +- Store sensitive tokens in the vault system +- Validate and sanitize data in your n8n workflow +- Consider rate limiting in your n8n instance + +## Troubleshooting + +### Common Issues + +1. **Webhook not responding**: Check n8n workflow is active and webhook URL is correct +2. **Authentication errors**: Verify auth_token is valid and properly stored in vault +3. **JSON parsing errors**: Ensure n8n workflow returns valid JSON + +### Debug Logging + +The connector provides detailed logging: +- Request payload sent to webhook +- Response status and content +- Processing results + +Check FlowSint logs for detailed debugging information. + +## Resources + +- [n8n Webhook Documentation](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.webhook/) +- [Respond to Webhook Documentation](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.respondtowebhook/) +- [n8n HTTP Request Node](https://docs.n8n.io/integrations/builtin/core-nodes/n8n-nodes-base.httprequest/) +""" + @classmethod def icon(cls) -> str | None: return "n8n" diff --git a/flowsint-api/app/scanners/organizations/org_to_asn.py b/flowsint-api/app/scanners/organizations/org_to_asn.py index 71701de..4051155 100644 --- a/flowsint-api/app/scanners/organizations/org_to_asn.py +++ b/flowsint-api/app/scanners/organizations/org_to_asn.py @@ -108,35 +108,24 @@ class OrgToAsnScanner(Scanner): if result_asn.number == 0: continue - query = """ - MERGE (org:organization {name: $org_name}) - SET org.sketch_id = $sketch_id, - org.label = $org_name, - org.caption = $org_name, - org.type = "organization" - - MERGE (asn:asn {number: $asn_number}) - SET asn.sketch_id = $sketch_id, - asn.name = $asn_name, - asn.country = $asn_country, - asn.label = $asn_label, - asn.caption = $asn_caption, - asn.type = "asn" - - MERGE (org)-[:BELONGS_TO {sketch_id: $sketch_id}]->(asn) - """ - if self.neo4j_conn: - self.neo4j_conn.query(query, { - "org_name": input_org.name, - "asn_number": result_asn.number, - "asn_name": result_asn.name, - "asn_country": result_asn.country, - "asn_label": f"AS{result_asn.number}", - "asn_caption": f"AS{result_asn.number} - {result_asn.name}", - "sketch_id": self.sketch_id, - }) - Logger.graph_append(self.sketch_id, {"message": f"Found for {input_org.name} -> ASN {result_asn.number}"}) + # Create organization node + self.create_node('organization', 'name', input_org.name, + caption=input_org.name, type="organization") + + # Create ASN node + self.create_node('asn', 'number', result_asn.number, + name=result_asn.name, + country=result_asn.country, + label=f"AS{result_asn.number}", + caption=f"AS{result_asn.number} - {result_asn.name}", + type="asn") + + # Create relationship + self.create_relationship('organization', 'name', input_org.name, + 'asn', 'number', result_asn.number, 'BELONGS_TO') + + self.log_graph_message(f"Found for {input_org.name} -> ASN {result_asn.number}") return results diff --git a/flowsint-api/app/scanners/organizations/to_infos.py b/flowsint-api/app/scanners/organizations/to_infos.py index a4147bd..07082e3 100644 --- a/flowsint-api/app/scanners/organizations/to_infos.py +++ b/flowsint-api/app/scanners/organizations/to_infos.py @@ -221,164 +221,99 @@ class OrgToInfosScanner(Scanner): for org in results: # Create or update the organization node with all SIRENE properties - self.neo4j_conn.query(""" - MERGE (o:Organization {name: $name, country: $country}) - SET o.siren = $siren, - o.siege_siret = $siret, - o.nom_complet = $nom_complet, - o.nom_raison_sociale = $nom_raison_sociale, - o.sigle = $sigle, - o.nombre_etablissements = $nombre_etablissements, - o.nombre_etablissements_ouverts = $nombre_etablissements_ouverts, - o.activite_principale = $activite_principale, - o.section_activite_principale = $section_activite_principale, - o.categorie_entreprise = $categorie_entreprise, - o.annee_categorie_entreprise = $annee_categorie_entreprise, - o.caractere_employeur = $caractere_employeur, - o.tranche_effectif_salarie = $tranche_effectif_salarie, - o.annee_tranche_effectif_salarie = $annee_tranche_effectif_salarie, - o.date_creation = $date_creation, - o.date_fermeture = $date_fermeture, - o.date_mise_a_jour = $date_mise_a_jour, - o.date_mise_a_jour_insee = $date_mise_a_jour_insee, - o.date_mise_a_jour_rne = $date_mise_a_jour_rne, - o.nature_juridique = $nature_juridique, - o.etat_administratif = $etat_administratif, - o.statut_diffusion = $statut_diffusion, - o.sketch_id = $sketch_id, - o.label = $name, - o.caption = $name, - o.type = 'organization' - """, { - "name": org.name, - "country": "FR", - "siren": org.siren, - "siret": org.siege_siret, - "nom_complet": org.nom_complet, - "nom_raison_sociale": org.nom_raison_sociale, - "sigle": org.sigle, - "nombre_etablissements": org.nombre_etablissements, - "nombre_etablissements_ouverts": org.nombre_etablissements_ouverts, - "activite_principale": org.activite_principale, - "section_activite_principale": org.section_activite_principale, - "categorie_entreprise": org.categorie_entreprise, - "annee_categorie_entreprise": org.annee_categorie_entreprise, - "caractere_employeur": org.caractere_employeur, - "tranche_effectif_salarie": org.tranche_effectif_salarie, - "annee_tranche_effectif_salarie": org.annee_tranche_effectif_salarie, - "date_creation": org.date_creation, - "date_fermeture": org.date_fermeture, - "date_mise_a_jour": org.date_mise_a_jour, - "date_mise_a_jour_insee": org.date_mise_a_jour_insee, - "date_mise_a_jour_rne": org.date_mise_a_jour_rne, - "nature_juridique": org.nature_juridique, - "etat_administratif": org.etat_administratif, - "statut_diffusion": org.statut_diffusion, - "sketch_id": self.sketch_id, - }) + org_key = f"{org.name}_FR" + self.create_node('Organization', 'org_id', org_key, + name=org.name, + country="FR", + siren=org.siren, + siege_siret=org.siege_siret, + nom_complet=org.nom_complet, + nom_raison_sociale=org.nom_raison_sociale, + sigle=org.sigle, + nombre_etablissements=org.nombre_etablissements, + nombre_etablissements_ouverts=org.nombre_etablissements_ouverts, + activite_principale=org.activite_principale, + section_activite_principale=org.section_activite_principale, + categorie_entreprise=org.categorie_entreprise, + annee_categorie_entreprise=org.annee_categorie_entreprise, + caractere_employeur=org.caractere_employeur, + tranche_effectif_salarie=org.tranche_effectif_salarie, + annee_tranche_effectif_salarie=org.annee_tranche_effectif_salarie, + date_creation=org.date_creation, + date_fermeture=org.date_fermeture, + date_mise_a_jour=org.date_mise_a_jour, + date_mise_a_jour_insee=org.date_mise_a_jour_insee, + date_mise_a_jour_rne=org.date_mise_a_jour_rne, + nature_juridique=org.nature_juridique, + etat_administratif=org.etat_administratif, + statut_diffusion=org.statut_diffusion, + caption=org.name, type='organization') if org.siren: - Logger.graph_append(self.sketch_id, {"message": f"{org.name}: SIREN {org.siren} -> {org.name}"}) + self.log_graph_message(f"{org.name}: SIREN {org.siren} -> {org.name}") # Add SIRET as identifier if available if org.siege_siret: - Logger.graph_append(self.sketch_id, {"message": f"{org.name}: SIRET {org.siege_siret} -> {org.name}"}) + self.log_graph_message(f"{org.name}: SIRET {org.siege_siret} -> {org.name}") # Add dirigeants (leaders) as Individual nodes with relationships if org.dirigeants: for dirigeant in org.dirigeants: - self.neo4j_conn.query(""" - MERGE (i:Individual {full_name: $full_name}) - SET i.first_name = $first_name, - i.last_name = $last_name, - i.birth_date = $birth_date, - i.gender = $gender, - i.sketch_id = $sketch_id, - i.label = $full_name, - i.caption = $full_name, - i.type = 'individual' - WITH i - MATCH (o:Organization {name: $org_name, country: $org_country}) - MERGE (o)-[:HAS_LEADER {sketch_id: $sketch_id}]->(i) - """, { - "full_name": dirigeant.full_name, - "first_name": dirigeant.first_name, - "last_name": dirigeant.last_name, - "birth_date": dirigeant.birth_date, - "gender": dirigeant.gender, - "sketch_id": self.sketch_id, - "org_name": org.name, - "org_country": "FR", - }) - Logger.graph_append(self.sketch_id, {"message": f"{org.name}: HAS_LEADER -> {dirigeant.full_name}"}) + self.create_node('Individual', 'full_name', dirigeant.full_name, + first_name=dirigeant.first_name, + last_name=dirigeant.last_name, + birth_date=dirigeant.birth_date, + gender=dirigeant.gender, + caption=dirigeant.full_name, type='individual') + + self.create_relationship('Organization', 'org_id', org_key, + 'Individual', 'full_name', dirigeant.full_name, 'HAS_LEADER') + self.log_graph_message(f"{org.name}: HAS_LEADER -> {dirigeant.full_name}") # Add siege address as PhysicalAddress node if available if org.siege_geo_adresse: address = org.siege_geo_adresse - self.neo4j_conn.query(""" - MERGE (a:PhysicalAddress {address: $address, city: $city, country: $country}) - SET a.zip = $zip, - a.latitude = $latitude, - a.longitude = $longitude, - a.sketch_id = $sketch_id, - a.label = $label, - a.caption = $caption, - a.type = 'location' - WITH a - MATCH (o:Organization {name: $org_name, country: $org_country}) - MERGE (o)-[:HAS_ADDRESS {sketch_id: $sketch_id}]->(a) - """, { - "address": address.address, - "city": address.city, - "country": address.country, - "zip": address.zip, - "latitude": address.latitude, - "longitude": address.longitude, - "sketch_id": self.sketch_id, - "label": f"{address.address}, {address.city}", - "caption": f"{address.address}, {address.city}", - "org_name": org.name, - "org_country": "FR", - }) - Logger.graph_append(self.sketch_id, {"message": f"{org.name}: HAS_ADDRESS -> {address.address}, {address.city}"}) + address_key = f"{address.address}_{address.city}_{address.country}" + self.create_node('PhysicalAddress', 'address_id', address_key, + address=address.address, + city=address.city, + country=address.country, + zip=address.zip, + latitude=address.latitude, + longitude=address.longitude, + label=f"{address.address}, {address.city}", + caption=f"{address.address}, {address.city}", + type='location') + + self.create_relationship('Organization', 'org_id', org_key, + 'PhysicalAddress', 'address_id', address_key, 'HAS_ADDRESS') + self.log_graph_message(f"{org.name}: HAS_ADDRESS -> {address.address}, {address.city}") # Add siege location as Location node if coordinates are available but no PhysicalAddress elif org.siege_latitude and org.siege_longitude: - self.neo4j_conn.query(""" - MERGE (l:Location {latitude: $latitude, longitude: $longitude}) - SET l.address = $address, - l.city = $city, - l.country = $country, - l.zip = $zip, - l.sketch_id = $sketch_id, - l.label = $label, - l.caption = $caption, - l.type = 'location' - WITH l - MATCH (o:Organization {name: $org_name, country: $org_country}) - MERGE (o)-[:LOCATED_AT {sketch_id: $sketch_id}]->(l) - """, { - "latitude": float(org.siege_latitude), - "longitude": float(org.siege_longitude), - "address": org.siege_adresse, - "city": org.siege_libelle_commune, - "country": "FR", - "zip": org.siege_code_postal, - "sketch_id": self.sketch_id, - "label": f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}", - "caption": f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}", - "org_name": org.name, - "org_country": "FR", - }) - Logger.graph_append(self.sketch_id, {"message": f"{org.name}: LOCATED_AT -> {org.siege_libelle_commune or 'Unknown'}"}) + location_key = f"{org.siege_latitude}_{org.siege_longitude}" + self.create_node('Location', 'location_id', location_key, + latitude=float(org.siege_latitude), + longitude=float(org.siege_longitude), + address=org.siege_adresse, + city=org.siege_libelle_commune, + country="FR", + zip=org.siege_code_postal, + label=f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}", + caption=f"{org.siege_adresse or 'Unknown'}, {org.siege_libelle_commune or 'Unknown'}", + type='location') + + self.create_relationship('Organization', 'org_id', org_key, + 'Location', 'location_id', location_key, 'LOCATED_AT') + self.log_graph_message(f"{org.name}: LOCATED_AT -> {org.siege_libelle_commune or 'Unknown'}") # Add activity codes as Activity nodes if org.activite_principale: - Logger.graph_append(self.sketch_id, {"message": f"{org.name}: HAS_ACTIVITY -> {org.activite_principale}"}) + self.log_graph_message(f"{org.name}: HAS_ACTIVITY -> {org.activite_principale}") # Add legal nature as LegalNature node if org.nature_juridique: - Logger.graph_append(self.sketch_id, {"message": f"{org.name}: HAS_LEGAL_NATURE -> {org.nature_juridique}"}) + self.log_graph_message(f"{org.name}: HAS_LEGAL_NATURE -> {org.nature_juridique}") return results diff --git a/flowsint-api/app/scanners/phones/ignorant.py b/flowsint-api/app/scanners/phones/ignorant.py index c08811e..7110975 100644 --- a/flowsint-api/app/scanners/phones/ignorant.py +++ b/flowsint-api/app/scanners/phones/ignorant.py @@ -97,24 +97,14 @@ class IgnorantScanner(Scanner): for result in results: if "error" not in result and "platforms" in result: - # Create phone number node - phone_query = """ - MERGE (phone:phone {number: $number}) - SET phone.sketch_id = $sketch_id, - phone.label = $number, - phone.caption = $number, - phone.type = "phone" - """ - self.neo4j_conn.query(phone_query, { - "number": result["number"], - "sketch_id": self.sketch_id - }) + self.create_node('phone', 'number', result["number"], + caption=result["number"], type='phone') # Create platform relationships for platform_result in result["platforms"]: if platform_result and isinstance(platform_result, dict): platform_name = platform_result.get("platform", "unknown") - Logger.graph_append(self.sketch_id, {"message": f"Phone {result['number']} found on {platform_name}"}) + self.log_graph_message(f"Phone {result['number']} found on {platform_name}") return results diff --git a/flowsint-api/app/scanners/registry.py b/flowsint-api/app/scanners/registry.py index 2d9ff0e..6cd1eda 100644 --- a/flowsint-api/app/scanners/registry.py +++ b/flowsint-api/app/scanners/registry.py @@ -1,4 +1,5 @@ -from typing import Dict, Type +import inspect +from typing import Dict, Type, List from app.scanners.base import Scanner from app.scanners.domains.subdomains import SubdomainScanner from app.scanners.domains.whois import WhoisScanner @@ -15,6 +16,7 @@ from app.scanners.crypto.wallet_to_transactions import CryptoWalletAddressToTran from app.scanners.crypto.wallet_to_nfts import CryptoWalletAddressToNFTs from app.scanners.domains.to_website import DomainToWebsiteScanner from app.scanners.websites.to_crawler import WebsiteToCrawler +from app.scanners.websites.to_links import WebsiteToLinks from app.scanners.websites.to_domain import WebsiteToDomainScanner from app.scanners.emails.to_gravatar import EmailToGravatarScanner from app.scanners.emails.to_leaks import EmailToBreachesScanner @@ -33,79 +35,23 @@ class ScannerRegistry: @classmethod def scanner_exists(cls, name: str) -> bool: - if name not in cls._scanners: - return False - return True + return name in cls._scanners @classmethod - def get_scanner(cls, name: str, sketch_id:str, scan_id: str, **kwargs) -> Scanner: + def get_scanner(cls, name: str, sketch_id: str, scan_id: str, **kwargs) -> Scanner: if name not in cls._scanners: raise Exception(f"Scanner '{name}' not found") return cls._scanners[name](sketch_id=sketch_id, scan_id=scan_id, **kwargs) @classmethod - def list(cls) -> Dict[str, Dict[str, str]]: + def _create_scanner_metadata(cls, scanner: Type[Scanner]) -> Dict[str, str]: + """Helper method to create scanner metadata dictionary.""" return { - name: { - "class_name": scanner.__name__, - "name": scanner.name(), - "module": scanner.__module__, - "doc": scanner.__doc__, - "category": scanner.category(), - "inputs": scanner.input_schema(), - "outputs": scanner.output_schema(), - "params": {}, - "params_schema": scanner.get_params_schema(), - "required_params": scanner.required_params(), - "icon": scanner.icon(), - } - for name, scanner in cls._scanners.items() - } - - @classmethod - def list_by_categories(cls) -> Dict[str, Dict[str, str]]: - scanners_by_category = {} - for _, scanner in cls._scanners.items(): - category = scanner.category() - if category not in scanners_by_category: - scanners_by_category[category] = [] - scanners_by_category[category].append({ - "class_name": scanner.__name__, - "name": scanner.name(), - "module": scanner.__module__, - "doc": scanner.__doc__, - "category": category, - "inputs": scanner.input_schema(), - "outputs": scanner.output_schema(), - "params": {}, - "params_schema": scanner.get_params_schema(), - "required_params": scanner.required_params(), - "icon": scanner.icon(), - }) - return scanners_by_category - - @classmethod - def list_by_input_type(cls, input_type: str) -> Dict[str, Dict[str, str]]: - if input_type.lower() == "any": - return [{ - "class_name": scanner.__name__, - "name": scanner.name(), - "module": scanner.__module__, - "doc": scanner.__doc__, - "category": scanner.category(), - "inputs": scanner.input_schema(), - "outputs": scanner.output_schema(), - "params": {}, - "params_schema": scanner.get_params_schema(), - "required_params": scanner.required_params(), - "icon": scanner.icon(), - } for _, scanner in cls._scanners.items()] - - return [{ "class_name": scanner.__name__, "name": scanner.name(), "module": scanner.__module__, - "doc": scanner.__doc__, + "description": scanner.__doc__, + "documentation": inspect.cleandoc(scanner.documentation()), "category": scanner.category(), "inputs": scanner.input_schema(), "outputs": scanner.output_schema(), @@ -113,8 +59,39 @@ class ScannerRegistry: "params_schema": scanner.get_params_schema(), "required_params": scanner.required_params(), "icon": scanner.icon(), - } for _, scanner in cls._scanners.items() if scanner.input_schema()["type"].lower() in ["any", input_type.lower()]] + } + @classmethod + def list(cls) -> Dict[str, Dict[str, str]]: + return { + name: cls._create_scanner_metadata(scanner) + for name, scanner in cls._scanners.items() + } + + @classmethod + def list_by_categories(cls) -> Dict[str, List[Dict[str, str]]]: + scanners_by_category = {} + for _, scanner in cls._scanners.items(): + category = scanner.category() + if category not in scanners_by_category: + scanners_by_category[category] = [] + scanners_by_category[category].append(cls._create_scanner_metadata(scanner)) + return scanners_by_category + + @classmethod + def list_by_input_type(cls, input_type: str) -> List[Dict[str, str]]: + input_type_lower = input_type.lower() + + if input_type_lower == "any": + return [cls._create_scanner_metadata(scanner) for scanner in cls._scanners.values()] + + return [ + cls._create_scanner_metadata(scanner) + for scanner in cls._scanners.values() + if scanner.input_schema()["type"].lower() in ["any", input_type_lower] + ] + +# Register all scanners ScannerRegistry.register(ReverseResolveScanner) ScannerRegistry.register(ResolveScanner) ScannerRegistry.register(SubdomainScanner) @@ -130,12 +107,12 @@ ScannerRegistry.register(CryptoWalletAddressToTransactions) ScannerRegistry.register(CryptoWalletAddressToNFTs) ScannerRegistry.register(DomainToWebsiteScanner) ScannerRegistry.register(WebsiteToCrawler) +ScannerRegistry.register(WebsiteToLinks) ScannerRegistry.register(WebsiteToDomainScanner) ScannerRegistry.register(EmailToGravatarScanner) ScannerRegistry.register(EmailToBreachesScanner) ScannerRegistry.register(IndividualToOrgScanner) ScannerRegistry.register(OrgToInfosScanner) -ScannerRegistry.register(IndividualToOrgScanner) ScannerRegistry.register(WebsiteToWebtrackersScanner) ScannerRegistry.register(N8nConnector) diff --git a/flowsint-api/app/scanners/socials/maigret.py b/flowsint-api/app/scanners/socials/maigret.py index 2944120..d54327d 100644 --- a/flowsint-api/app/scanners/socials/maigret.py +++ b/flowsint-api/app/scanners/socials/maigret.py @@ -121,40 +121,28 @@ class MaigretScanner(Scanner): return results for profile in results: - Logger.graph_append(self.sketch_id, {"message": f"{profile.username} -> account found on {profile.platform}"}) - self.neo4j_conn.query(""" - MERGE (p:social_profile {profile_url: $profile_url}) - SET p.username = $username, - p.platform = $platform, - p.profile_picture_url = $picture, - p.bio = $bio, - p.followers_count = $followers, - p.following_count = $following, - p.posts_count = $posts, - p.label = $label, - p.caption = $caption, - p.color = $color, - p.type = $type, - p.sketch_id = $sketch_id - - MERGE (i:username {username: $username}) - SET i.sketch_id = $sketch_id - MERGE (i)-[:HAS_SOCIAL_ACCOUNT {sketch_id: $sketch_id}]->(p) - """, { - "profile_url": profile.profile_url, - "username": profile.username, - "platform": profile.platform, - "picture": profile.profile_picture_url, - "bio": profile.bio, - "followers": profile.followers_count, - "following": profile.following_count, - "posts": profile.posts_count, - "label": f"{profile.platform}:{profile.username}", - "caption": f"{profile.platform}:{profile.username}", - "color": "#1DA1F2", - "type": "social_profile", - "sketch_id": self.sketch_id - }) + # Create social profile node + self.create_node('social_profile', 'profile_url', profile.profile_url, + username=profile.username, + platform=profile.platform, + profile_picture_url=profile.profile_picture_url, + bio=profile.bio, + followers_count=profile.followers_count, + following_count=profile.following_count, + posts_count=profile.posts_count, + label=f"{profile.platform}:{profile.username}", + caption=f"{profile.platform}:{profile.username}", + color="#1DA1F2", + type="social_profile") + + # Create username node + self.create_node('username', 'username', profile.username, type='username') + + # Create relationship + self.create_relationship('username', 'username', profile.username, + 'social_profile', 'profile_url', profile.profile_url, 'HAS_SOCIAL_ACCOUNT') + + self.log_graph_message(f"{profile.username} -> account found on {profile.platform}") return results diff --git a/flowsint-api/app/scanners/socials/sherlock.py b/flowsint-api/app/scanners/socials/sherlock.py index e8c86e9..873705b 100644 --- a/flowsint-api/app/scanners/socials/sherlock.py +++ b/flowsint-api/app/scanners/socials/sherlock.py @@ -94,22 +94,10 @@ class SherlockScanner(Scanner): return results for social in results: - # Create or update social account node - social_query = """ - MERGE (social:social {username: $username, platform: $platform}) - SET social.url = $url, - social.sketch_id = $sketch_id, - social.label = $username, - social.caption = $platform, - social.type = "social" - """ - self.neo4j_conn.query(social_query, { - "username": social.username, - "platform": social.platform, - "url": social.url, - "sketch_id": self.sketch_id - }) - Logger.graph_append(self.sketch_id, {"message": f"Found social account: {social.username} on {social.platform}"}) + self.create_node('social', 'username', social.username, + platform=social.platform, url=social.url, + caption=social.platform, type='social') + self.log_graph_message(f"Found social account: {social.username} on {social.platform}") return results diff --git a/flowsint-api/app/scanners/websites/to_crawler.py b/flowsint-api/app/scanners/websites/to_crawler.py index edd6133..916de7e 100644 --- a/flowsint-api/app/scanners/websites/to_crawler.py +++ b/flowsint-api/app/scanners/websites/to_crawler.py @@ -105,59 +105,25 @@ class WebsiteToCrawler(Scanner): website_url = str(input_website.url) # Create website node - website_query = """ - MERGE (website:website {url: $website_url}) - SET website.sketch_id = $sketch_id, - website.label = $website_url, - website.caption = $website_url, - website.type = "website" - """ - if self.neo4j_conn: - self.neo4j_conn.query(website_query, { - "website_url": website_url, - "sketch_id": self.sketch_id, - }) + self.create_node('website', 'url', website_url, + caption=website_url, type='website') # Create email nodes and relationships for email in result["emails"]: - email_query = """ - MERGE (email:email {email: $email_address}) - SET email.sketch_id = $sketch_id, - email.label = $email_address, - email.caption = $email_address, - email.type = "email" - - MERGE (website:website {url: $website_url}) - MERGE (website)-[:HAS_EMAIL {sketch_id: $sketch_id}]->(email) - """ - - self.neo4j_conn.query(email_query, { - "email_address": email.email, - "website_url": website_url, - "sketch_id": self.sketch_id, - }) - Logger.graph_append(self.sketch_id, {"message": f"Found email {email.email} for website {website_url}"}) + self.create_node('email', 'email', email.email, + caption=email.email, type='email') + self.create_relationship('website', 'url', website_url, + 'email', 'email', email.email, 'HAS_EMAIL') + self.log_graph_message(f"Found email {email.email} for website {website_url}") # Create phone nodes and relationships for phone in result["phones"]: - phone_query = """ - MERGE (phone:phone {number: $phone_number}) - SET phone.sketch_id = $sketch_id, - phone.label = $phone_number, - phone.caption = $phone_number, - phone.type = "phone" - - MERGE (website:website {url: $website_url}) - MERGE (website)-[:HAS_PHONE {sketch_id: $sketch_id}]->(phone) - """ - - self.neo4j_conn.query(phone_query, { - "phone_number": phone.number, - "website_url": website_url, - "sketch_id": self.sketch_id, - }) - Logger.graph_append(self.sketch_id, {"message": f"Found phone {phone.number} for website {website_url}"}) + self.create_node('phone', 'number', phone.number, + caption=phone.number, type='phone') + self.create_relationship('website', 'url', website_url, + 'phone', 'number', phone.number, 'HAS_PHONE') + self.log_graph_message(f"Found phone {phone.number} for website {website_url}") return results diff --git a/flowsint-api/app/scanners/websites/to_links.py b/flowsint-api/app/scanners/websites/to_links.py new file mode 100644 index 0000000..c67347c --- /dev/null +++ b/flowsint-api/app/scanners/websites/to_links.py @@ -0,0 +1,200 @@ +from typing import List, Union +from urllib.parse import urlparse +from app.scanners.base import Scanner +from app.types.website import Website +from app.core.logger import Logger +from reconspread import Crawler + +class WebsiteToLinks(Scanner): + """From website to spread crawler that extracts domains and internal/external links.""" + + # Define types as class attributes - base class handles schema generation automatically + InputType = List[Website] + OutputType = List[Website] + + @classmethod + def name(cls) -> str: + return "to_links" + + @classmethod + def category(cls) -> str: + return "Website" + + @classmethod + def key(cls) -> str: + return "url" + + def extract_domain(self, url: str) -> str: + """Extract domain from URL.""" + try: + parsed_url = urlparse(url) + return parsed_url.netloc + except Exception: + return "" + + def preprocess(self, data: Union[List[str], List[dict], InputType]) -> InputType: + cleaned: InputType = [] + for item in data: + website_obj = None + if isinstance(item, str): + website_obj = Website(url=item) + elif isinstance(item, dict) and "url" in item: + website_obj = Website(url=item["url"]) + elif isinstance(item, Website): + website_obj = item + if website_obj: + cleaned.append(website_obj) + return cleaned + + async def scan(self, data: InputType) -> OutputType: + """Crawl websites using reconspread to extract internal and external links.""" + results = [] + + for website in data: + try: + Logger.info(self.sketch_id, {"message": f"Starting reconspread crawl of {str(website.url)}"}) + + # Extract main domain from input website (needed in callback) + main_domain = self.extract_domain(str(website.url)) + + # Create main website and domain nodes upfront + if self.neo4j_conn: + self.create_node('website', 'url', str(website.url), + caption=str(website.url), type='website') + if main_domain: + self.create_node('domain', 'name', main_domain, + caption=main_domain, type='domain') + self.create_relationship('website', 'url', str(website.url), + 'domain', 'name', main_domain, 'BELONGS_TO_DOMAIN') + self.log_graph_message(f"Website {str(website.url)} belongs to domain {main_domain}") + + # Store discovered URLs + internal_urls = [] + external_urls = [] + external_domains = set() + + def url_handler(url, is_external=False): + """Custom callback to handle URLs as they're discovered.""" + if is_external: + external_urls.append(url) + domain = self.extract_domain(url) + if domain: + external_domains.add(domain) + # Create external website node immediately + if self.neo4j_conn: + self.create_node('website', 'url', url, + caption=url, type='website') + self.create_relationship('website', 'url', str(website.url), + 'website', 'url', url, 'LINKS_TO') + self.log_graph_message(f"Website {str(website.url)} links to external website {url}") + + # Create external domain node and link external website to its domain + if domain != main_domain: + self.create_node('domain', 'name', domain, + caption=domain, type='domain') + self.create_relationship('website', 'url', url, + 'domain', 'name', domain, 'BELONGS_TO_DOMAIN') + self.create_relationship('website', 'url', str(website.url), + 'domain', 'name', domain, 'LINKS_TO_DOMAIN') + self.log_graph_message(f"External website {url} belongs to domain {domain}") + self.log_graph_message(f"Website {str(website.url)} links to external domain {domain}") + Logger.info(self.sketch_id, {"message": f"[EXTERNAL] Found: {url} -> Domain: {domain}"}) + else: + internal_urls.append(url) + # Create internal website node immediately + if self.neo4j_conn and url != str(website.url): # Don't create duplicate of main website + self.create_node('website', 'url', url, + caption=url, type='website') + self.create_relationship('website', 'url', str(website.url), + 'website', 'url', url, 'LINKS_TO') + self.log_graph_message(f"Website {str(website.url)} links to internal website {url}") + + # Also link internal websites to main domain + if main_domain: + self.create_relationship('website', 'url', url, + 'domain', 'name', main_domain, 'BELONGS_TO_DOMAIN') + Logger.info(self.sketch_id, {"message": f"[INTERNAL] Found: {url}"}) + + # Create crawler with custom callback + crawler = Crawler( + url=str(website.url), + recursive=True, + same_domain_only=False, + verbose=False, # Disable default verbose output + _on_result_callback=url_handler # Use our custom handler + ) + + # Perform the crawl + crawler.fetch() + crawler.extract_urls() + + # Get final results (backup in case callback missed anything) + crawl_results = crawler.get_results() + + # Ensure we have all internal URLs + for url in crawl_results.internal: + if url not in internal_urls: + internal_urls.append(url) + + # Ensure we have all external URLs and domains + for url in crawl_results.external: + if url not in external_urls: + external_urls.append(url) + domain = self.extract_domain(url) + if domain: + external_domains.add(domain) + + website_result = { + "website": str(website.url), + "main_domain": main_domain, + "internal_urls": internal_urls, + "external_urls": external_urls, + "external_domains": list(external_domains) + } + + # Log results + Logger.info(self.sketch_id, { + "message": f"Spread crawl completed for {str(website.url)}: " + f"Main domain: {main_domain}, " + f"{len(internal_urls)} internal URLs, " + f"{len(external_urls)} external URLs, " + f"{len(external_domains)} external domains found." + }) + + results.append(website_result) + + except Exception as e: + # Log error but continue with other websites + Logger.error(self.sketch_id, {"message": f"Error crawling {str(website.url)}: {str(e)}"}) + + # Still create main website and domain nodes even on error + main_domain = self.extract_domain(str(website.url)) + if self.neo4j_conn: + self.create_node('website', 'url', str(website.url), + caption=str(website.url), type='website') + if main_domain: + self.create_node('domain', 'name', main_domain, + caption=main_domain, type='domain') + self.create_relationship('website', 'url', str(website.url), + 'domain', 'name', main_domain, 'BELONGS_TO_DOMAIN') + self.log_graph_message(f"Website {str(website.url)} belongs to domain {main_domain}") + + # Add empty result for failed website + results.append({ + "website": str(website.url), + "main_domain": main_domain, + "internal_urls": [], + "external_urls": [], + "external_domains": [] + }) + continue + + return results + + def postprocess(self, results: OutputType, original_input: InputType) -> OutputType: + # Neo4j nodes and relationships are created in real-time during the callback + # No additional processing needed here + return results + +InputType = WebsiteToLinks.InputType +OutputType = WebsiteToLinks.OutputType \ No newline at end of file diff --git a/flowsint-api/app/types/whois.py b/flowsint-api/app/types/whois.py index 92c0361..98ee858 100644 --- a/flowsint-api/app/types/whois.py +++ b/flowsint-api/app/types/whois.py @@ -4,6 +4,7 @@ from app.types.email import Email class Whois(BaseModel): """Represents WHOIS domain registration information.""" + domain: str = Field(..., description="Domain name", title="Domain") registrar: Optional[str] = Field(None, description="Domain registrar name", title="Registrar") org: Optional[str] = Field(None, description="Organization name associated with the domain", title="Organization") city: Optional[str] = Field(None, description="City where the domain is registered", title="City") diff --git a/flowsint-api/app/utils.py b/flowsint-api/app/utils.py index 40f6bf1..b1e02a8 100644 --- a/flowsint-api/app/utils.py +++ b/flowsint-api/app/utils.py @@ -7,8 +7,11 @@ from urllib.parse import urlparse import re import ssl import socket -from typing import Dict, Any, List, Optional, Type -from pydantic import BaseModel, ValidationError, Field, create_model +from typing import Dict, Any, List, Type +from pydantic import BaseModel +import inspect +from typing import Any, Dict, Type +from pydantic import BaseModel, TypeAdapter def is_valid_ip(address: str) -> bool: try: @@ -146,7 +149,7 @@ def extract_input_schema_transform(model: Type[BaseModel]) -> Dict[str, Any]: "class_name": model.__name__, "name": model.__name__, "module": model.__module__, - "doc": model.__doc__ or "", + "description": inspect.cleandoc(model.__doc__ or ""), "outputs": { "type": type_name, "properties": [ diff --git a/flowsint-api/requirements.txt b/flowsint-api/requirements.txt index f8f9484..64d792d 100644 --- a/flowsint-api/requirements.txt +++ b/flowsint-api/requirements.txt @@ -32,4 +32,6 @@ docker git+https://github.com/soxoj/maigret git+https://github.com/reconurge/recontrack.git git+https://github.com/reconurge/reconcrawl.git -pytest-asyncio \ No newline at end of file +git+https://github.com/reconurge/reconspread.git +pytest-asyncio +stix2 \ No newline at end of file diff --git a/flowsint-api/tests/insert_command.py b/flowsint-api/tests/insert_command.py new file mode 100644 index 0000000..f6cf918 --- /dev/null +++ b/flowsint-api/tests/insert_command.py @@ -0,0 +1,28 @@ + +import sys +import os +import asyncio + +if __name__ == "__main__": + sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from app.types.domain import Domain +from app.types.ip import Ip +from app.scanners.domains.resolve import ResolveScanner + + +async def main(): + # Create test data + domains = [Domain(domain="adaltas.com")] + ips = [Ip(address='12.23.34.45'), Ip(address='56.67.78.89')] + + # Test the scanner + scanner = ResolveScanner("sketch_123", "scan_123") + + # Test the new KISS postprocess method + scanner.postprocess(ips[:1], domains) # Only use first IP to match domains length + + print("Postprocess test completed successfully!") + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/flowsint-api/tests/scanners/domains/resolve.py b/flowsint-api/tests/scanners/domains/resolve.py index 0dc39ae..1933871 100644 --- a/flowsint-api/tests/scanners/domains/resolve.py +++ b/flowsint-api/tests/scanners/domains/resolve.py @@ -87,7 +87,7 @@ def test_schemas(): class TestResolveInputOutputTypes: - """Test the new InputType/OutputType functionality for ResolveScanner""" + """Test the InputType/OutputType functionality for ResolveScanner""" def test_input_output_types_are_defined(self): """Test that InputType and OutputType are properly defined""" diff --git a/flowsint-api/tests/scanners/websites/to_links.py b/flowsint-api/tests/scanners/websites/to_links.py new file mode 100644 index 0000000..59ff0cf --- /dev/null +++ b/flowsint-api/tests/scanners/websites/to_links.py @@ -0,0 +1,162 @@ +import pytest +from unittest.mock import Mock, patch, call +from app.scanners.websites.to_links import WebsiteToLinks +from app.types.website import Website + + +class MockCrawlResults: + def __init__(self, internal=None, external=None): + self.internal = internal or [] + self.external = external or [] + + +class MockCrawler: + def __init__(self, url, recursive=True, verbose=False, _on_result_callback=None): + self.url = url + self.callback = _on_result_callback + + def fetch(self): + pass + + def extract_urls(self): + # Simulate callback calls + if self.callback: + self.callback("https://example.com/page1", is_external=False) + self.callback("https://example.com/page2", is_external=False) + self.callback("https://external.com/page", is_external=True) + self.callback("https://another-external.org/resource", is_external=True) + + def get_results(self): + return MockCrawlResults( + internal=["https://example.com/page1", "https://example.com/page2"], + external=["https://external.com/page", "https://another-external.org/resource"] + ) + + +@pytest.mark.asyncio +async def test_website_to_links_real_time_neo4j_creation(): + """Test that Neo4j nodes are created in real-time during the callback.""" + scanner = WebsiteToLinks(sketch_id="test", scan_id="test") + + # Mock neo4j connection and methods + scanner.neo4j_conn = Mock() + scanner.create_node = Mock() + scanner.create_relationship = Mock() + scanner.log_graph_message = Mock() + + # Test input + websites = [Website(url="https://example.com")] + + with patch('app.scanners.websites.to_links.Crawler', MockCrawler): + results = await scanner.scan(websites) + + # Verify main website and domain nodes were created upfront + scanner.create_node.assert_any_call('website', 'url', 'https://example.com', + caption='https://example.com', type='website') + scanner.create_node.assert_any_call('domain', 'name', 'example.com', + caption='example.com', type='domain') + + # Verify main website to domain relationship + scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com', + 'domain', 'name', 'example.com', 'BELONGS_TO_DOMAIN') + + # Verify internal website nodes were created in callback + scanner.create_node.assert_any_call('website', 'url', 'https://example.com/page1', + caption='https://example.com/page1', type='website') + scanner.create_node.assert_any_call('website', 'url', 'https://example.com/page2', + caption='https://example.com/page2', type='website') + + # Verify internal website relationships + scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com', + 'website', 'url', 'https://example.com/page1', 'LINKS_TO') + scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com', + 'website', 'url', 'https://example.com/page2', 'LINKS_TO') + + # Verify external website nodes were created in callback + scanner.create_node.assert_any_call('website', 'url', 'https://external.com/page', + caption='https://external.com/page', type='website') + scanner.create_node.assert_any_call('website', 'url', 'https://another-external.org/resource', + caption='https://another-external.org/resource', type='website') + + # Verify external domain nodes were created in callback + scanner.create_node.assert_any_call('domain', 'name', 'external.com', + caption='external.com', type='domain') + scanner.create_node.assert_any_call('domain', 'name', 'another-external.org', + caption='another-external.org', type='domain') + + # Verify main website to external website relationships + scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com', + 'website', 'url', 'https://external.com/page', 'LINKS_TO') + scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com', + 'website', 'url', 'https://another-external.org/resource', 'LINKS_TO') + + # Verify external website to domain relationships + scanner.create_relationship.assert_any_call('website', 'url', 'https://external.com/page', + 'domain', 'name', 'external.com', 'BELONGS_TO_DOMAIN') + scanner.create_relationship.assert_any_call('website', 'url', 'https://another-external.org/resource', + 'domain', 'name', 'another-external.org', 'BELONGS_TO_DOMAIN') + + # Verify main website to external domain relationships + scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com', + 'domain', 'name', 'external.com', 'LINKS_TO_DOMAIN') + scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com', + 'domain', 'name', 'another-external.org', 'LINKS_TO_DOMAIN') + + +@pytest.mark.asyncio +async def test_website_to_links_error_handling_with_neo4j(): + """Test that main nodes are still created even when crawling fails.""" + scanner = WebsiteToLinks(sketch_id="test", scan_id="test") + + # Mock neo4j connection and methods + scanner.neo4j_conn = Mock() + scanner.create_node = Mock() + scanner.create_relationship = Mock() + scanner.log_graph_message = Mock() + + # Mock crawler that raises an exception + def mock_crawler_error(*args, **kwargs): + raise Exception("Test error") + + websites = [Website(url="https://example.com")] + + with patch('app.scanners.websites.to_links.Crawler', mock_crawler_error): + results = await scanner.scan(websites) + + # Verify main website and domain nodes were still created despite error + scanner.create_node.assert_any_call('website', 'url', 'https://example.com', + caption='https://example.com', type='website') + scanner.create_node.assert_any_call('domain', 'name', 'example.com', + caption='example.com', type='domain') + + # Verify main website to domain relationship was created + scanner.create_relationship.assert_any_call('website', 'url', 'https://example.com', + 'domain', 'name', 'example.com', 'BELONGS_TO_DOMAIN') + + # Verify result structure + assert len(results) == 1 + result = results[0] + assert result["website"] == "https://example.com" + assert result["main_domain"] == "example.com" + assert result["internal_urls"] == [] + assert result["external_urls"] == [] + assert result["external_domains"] == [] + + +def test_postprocess_simplified(): + """Test that postprocess now just returns results as-is.""" + scanner = WebsiteToLinks(sketch_id="test", scan_id="test") + + original_input = [Website(url="https://example.com")] + results = [{ + "website": "https://example.com", + "main_domain": "example.com", + "internal_urls": ["https://example.com/page1"], + "external_urls": ["https://external.com/page"], + "external_domains": ["external.com"] + }] + + processed_results = scanner.postprocess(results, original_input) + + # Should just return the same results since Neo4j work is done in real-time + assert processed_results == results \ No newline at end of file diff --git a/flowsint-app/Dockerfile b/flowsint-app/Dockerfile new file mode 100644 index 0000000..dfdddce --- /dev/null +++ b/flowsint-app/Dockerfile @@ -0,0 +1,52 @@ +# Étape 1 – Build Vite + Electron +FROM node:20 AS builder + +WORKDIR /app + +# Copie et install +COPY package*.json ./ +RUN npm install + +COPY . . + +# Vérification de type + build +RUN npm run typecheck +RUN npm run build -y + +# Build de l'app avec electron-builder (non packagée) +RUN npm run build:unpack + +# Étape 2 – Image finale avec l'app prête +FROM node:20-slim + +# Dépendances d'Electron requises pour s'exécuter dans un conteneur +RUN apt-get update && apt-get install -y \ + libgtk-3-0 \ + libx11-xcb1 \ + libnss3 \ + libxcomposite1 \ + libxrandr2 \ + libasound2 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libcups2 \ + libdrm2 \ + libgbm1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxkbcommon0 \ + libxshmfence1 \ + libglu1-mesa \ + x11-xserver-utils \ + --no-install-recommends && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /app/dist ./dist +COPY --from=builder /app/out ./out +COPY --from=builder /app/node_modules ./node_modules +COPY --from=builder /app/package.json ./package.json + +CMD ["npx", "electron", "."] diff --git a/flowsint-app/electron.vite.config.ts b/flowsint-app/electron.vite.config.ts index bbd1388..f516f63 100644 --- a/flowsint-app/electron.vite.config.ts +++ b/flowsint-app/electron.vite.config.ts @@ -26,7 +26,12 @@ export default defineConfig({ }), enforce: 'pre' }, - react(), + react({ + babel: { + // plugins: ["babel-plugin-react-compiler"] + } + } + ), tailwindcss() ], resolve: { diff --git a/flowsint-app/package.json b/flowsint-app/package.json index 68071c5..7105db4 100644 --- a/flowsint-app/package.json +++ b/flowsint-app/package.json @@ -11,7 +11,7 @@ "icon": "build/icon.icns" }, "author": "DexterMorgan", - "homepage": "https://www.flowsint.fr/", + "homepage": "https://www.flowsint.io/", "scripts": { "format": "prettier --write .", "lint": "eslint . --ext .js,.jsx,.cjs,.mjs,.ts,.tsx,.cts,.mts --fix", @@ -61,7 +61,8 @@ "@radix-ui/react-tooltip": "^1.2.7", "@react-sigma/core": "^5.0.4", "@react-sigma/layout-forceatlas2": "^5.0.4", - "@tailwindcss/vite": "^3.4.1", + "@tailwindcss/typography": "^0.5.16", + "@tailwindcss/vite": "^4.1.11", "@tanstack/react-query": "^5.79.0", "@tanstack/react-table": "^8.21.3", "@tanstack/react-virtual": "^3.13.9", @@ -87,12 +88,14 @@ "@tiptap/react": "^2.12.0", "@tiptap/starter-kit": "^2.12.0", "@xyflow/react": "^12.6.4", + "babel-plugin-react-compiler": "^19.1.0-rc.2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", "cytoscape": "^3.32.0", "d3-force": "^3.0.0", - "date-fns": "^4.1.0", + "dagre-d3-react": "^0.2.4", + "date-fns": "^3.6.0", "electron-updater": "^6.1.7", "embla-carousel-react": "^8.6.0", "graphology": "^0.26.0", @@ -112,6 +115,7 @@ "react-medium-image-zoom": "^5.2.14", "react-resizable-panels": "^3.0.2", "react-syntax-highlighter": "^15.6.1", + "reagraph": "^4.25.0", "recharts": "^2.15.3", "remark-gfm": "^4.0.1", "sigma": "^3.0.2", diff --git a/flowsint-app/src/main/index.ts b/flowsint-app/src/main/index.ts index 65dcdd5..5d12cc3 100644 --- a/flowsint-app/src/main/index.ts +++ b/flowsint-app/src/main/index.ts @@ -1,6 +1,6 @@ import { electronApp, optimizer } from '@electron-toolkit/utils' -import { app } from 'electron' -import { initializeMainWindow } from './main-window' +import { app, ipcMain } from 'electron' +import { initializeMainWindow, getMainWindow } from './main-window' app.whenReady().then(() => { electronApp.setAppUserModelId('com.electron') @@ -20,3 +20,14 @@ app.on('window-all-closed', () => { app.quit() } }) + +// IPC handlers for window state +ipcMain.handle('get-window-state', () => { + const mainWindow = getMainWindow() + if (!mainWindow) return { isFullscreen: false, isMaximized: false } + + return { + isFullscreen: mainWindow.isFullScreen(), + isMaximized: mainWindow.isMaximized() + } +}) diff --git a/flowsint-app/src/main/main-window.ts b/flowsint-app/src/main/main-window.ts index bb03a9a..fca0951 100644 --- a/flowsint-app/src/main/main-window.ts +++ b/flowsint-app/src/main/main-window.ts @@ -18,12 +18,16 @@ export async function initializeMainWindow() { height: 770, show: false, autoHideMenuBar: true, - titleBarOverlay: true, + titleBarOverlay: { + color: '#292524', + symbolColor: '#ffffff', + height: 48 + }, frame: false, titleBarStyle: 'hiddenInset', trafficLightPosition: { - x: 9, - y: 9 + x: 12, + y: 14 }, backgroundColor: '#292524', webPreferences: { diff --git a/flowsint-app/src/preload/index.d.ts b/flowsint-app/src/preload/index.d.ts index 22bd445..7fb70da 100644 --- a/flowsint-app/src/preload/index.d.ts +++ b/flowsint-app/src/preload/index.d.ts @@ -3,7 +3,12 @@ import { ElectronAPI } from '@electron-toolkit/preload' declare global { interface Window { electron: ElectronAPI - api: unknown + api: { + getWindowState: () => Promise<{ + isFullscreen: boolean + isMaximized: boolean + }> + } } } diff --git a/flowsint-app/src/preload/index.ts b/flowsint-app/src/preload/index.ts index 2d18524..94096f2 100644 --- a/flowsint-app/src/preload/index.ts +++ b/flowsint-app/src/preload/index.ts @@ -1,8 +1,10 @@ -import { contextBridge } from 'electron' +import { contextBridge, ipcRenderer } from 'electron' import { electronAPI } from '@electron-toolkit/preload' // Custom APIs for renderer -const api = {} +const api = { + getWindowState: () => ipcRenderer.invoke('get-window-state') +} // Use `contextBridge` APIs to expose Electron APIs to // renderer only if context isolation is enabled, otherwise diff --git a/flowsint-app/src/renderer/index.html b/flowsint-app/src/renderer/index.html index c41e657..153b424 100644 --- a/flowsint-app/src/renderer/index.html +++ b/flowsint-app/src/renderer/index.html @@ -1,11 +1,11 @@ - + diff --git a/flowsint-app/src/renderer/public/icons/asn.svg b/flowsint-app/src/renderer/public/icons/asn.svg index e606699..7e46094 100644 --- a/flowsint-app/src/renderer/public/icons/asn.svg +++ b/flowsint-app/src/renderer/public/icons/asn.svg @@ -1,14 +1,14 @@ - - - + + + - - - - + + + + - ASN + ASN \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/breach.svg b/flowsint-app/src/renderer/public/icons/breach.svg index fd05d95..0f95168 100644 --- a/flowsint-app/src/renderer/public/icons/breach.svg +++ b/flowsint-app/src/renderer/public/icons/breach.svg @@ -1,10 +1 @@ - - - - - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/cidr.svg b/flowsint-app/src/renderer/public/icons/cidr.svg index 31e40f1..4d79ac3 100644 --- a/flowsint-app/src/renderer/public/icons/cidr.svg +++ b/flowsint-app/src/renderer/public/icons/cidr.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/credential.svg b/flowsint-app/src/renderer/public/icons/credential.svg index 3dd4e6d..8715e8b 100644 --- a/flowsint-app/src/renderer/public/icons/credential.svg +++ b/flowsint-app/src/renderer/public/icons/credential.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/cryptowallet.svg b/flowsint-app/src/renderer/public/icons/cryptowallet.svg index 0ea6034..1ba4736 100644 --- a/flowsint-app/src/renderer/public/icons/cryptowallet.svg +++ b/flowsint-app/src/renderer/public/icons/cryptowallet.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/default.svg b/flowsint-app/src/renderer/public/icons/default.svg index 9ab085b..32c2547 100644 --- a/flowsint-app/src/renderer/public/icons/default.svg +++ b/flowsint-app/src/renderer/public/icons/default.svg @@ -1,4 +1,4 @@ - - + + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/device.svg b/flowsint-app/src/renderer/public/icons/device.svg index 4586746..ab8dcfe 100644 --- a/flowsint-app/src/renderer/public/icons/device.svg +++ b/flowsint-app/src/renderer/public/icons/device.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/domain.svg b/flowsint-app/src/renderer/public/icons/domain.svg index f15f51c..e86b8a1 100644 --- a/flowsint-app/src/renderer/public/icons/domain.svg +++ b/flowsint-app/src/renderer/public/icons/domain.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/email.svg b/flowsint-app/src/renderer/public/icons/email.svg index 2449793..dee1304 100644 --- a/flowsint-app/src/renderer/public/icons/email.svg +++ b/flowsint-app/src/renderer/public/icons/email.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/individual.svg b/flowsint-app/src/renderer/public/icons/individual.svg index 533d34e..4dfba93 100644 --- a/flowsint-app/src/renderer/public/icons/individual.svg +++ b/flowsint-app/src/renderer/public/icons/individual.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/ip.svg b/flowsint-app/src/renderer/public/icons/ip.svg index e47b639..fae0006 100644 --- a/flowsint-app/src/renderer/public/icons/ip.svg +++ b/flowsint-app/src/renderer/public/icons/ip.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/location.svg b/flowsint-app/src/renderer/public/icons/location.svg index 9bcb7cb..a74bd6b 100644 --- a/flowsint-app/src/renderer/public/icons/location.svg +++ b/flowsint-app/src/renderer/public/icons/location.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/organization.svg b/flowsint-app/src/renderer/public/icons/organization.svg index bc4ca3a..7a16b5c 100644 --- a/flowsint-app/src/renderer/public/icons/organization.svg +++ b/flowsint-app/src/renderer/public/icons/organization.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/phone.svg b/flowsint-app/src/renderer/public/icons/phone.svg index e7e745b..f1ac6dc 100644 --- a/flowsint-app/src/renderer/public/icons/phone.svg +++ b/flowsint-app/src/renderer/public/icons/phone.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/phrase.svg b/flowsint-app/src/renderer/public/icons/phrase.svg index a25dd4f..3a8560a 100644 --- a/flowsint-app/src/renderer/public/icons/phrase.svg +++ b/flowsint-app/src/renderer/public/icons/phrase.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/subdomain.svg b/flowsint-app/src/renderer/public/icons/subdomain.svg index ca4b926..e86b8a1 100644 --- a/flowsint-app/src/renderer/public/icons/subdomain.svg +++ b/flowsint-app/src/renderer/public/icons/subdomain.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/weapon.svg b/flowsint-app/src/renderer/public/icons/weapon.svg index f24c947..19922e0 100644 --- a/flowsint-app/src/renderer/public/icons/weapon.svg +++ b/flowsint-app/src/renderer/public/icons/weapon.svg @@ -1,3 +1,3 @@ - + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/website.svg b/flowsint-app/src/renderer/public/icons/website.svg index f3f8540..a48b3f6 100644 --- a/flowsint-app/src/renderer/public/icons/website.svg +++ b/flowsint-app/src/renderer/public/icons/website.svg @@ -1,23 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/public/icons/whois.svg b/flowsint-app/src/renderer/public/icons/whois.svg index cfba43f..7fbab28 100644 --- a/flowsint-app/src/renderer/public/icons/whois.svg +++ b/flowsint-app/src/renderer/public/icons/whois.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/api/auth-service.ts b/flowsint-app/src/renderer/src/api/auth-service.ts index 531efce..03ac03d 100644 --- a/flowsint-app/src/renderer/src/api/auth-service.ts +++ b/flowsint-app/src/renderer/src/api/auth-service.ts @@ -1,19 +1,18 @@ import { useAuthStore } from '@/stores/auth-store'; import { fetchWithAuth } from './api'; -import { useBoundStore } from '@/stores/use-bound-store'; -interface RegisterData { +export interface RegisterData { username: string; email: string; password: string; } -interface LoginData { +export interface LoginData { username: string; password: string; } -interface AuthResponse { +export interface AuthResponse { access_token: string; token_type: string; user: { @@ -46,7 +45,6 @@ export const authService = { }); }, logout: () => { - useBoundStore.getState().tabs.clearTabs() useAuthStore.getState().logout() }, diff --git a/flowsint-app/src/renderer/src/components/analyses/analysis-editor.tsx b/flowsint-app/src/renderer/src/components/analyses/analysis-editor.tsx index 1458eb8..aba27da 100644 --- a/flowsint-app/src/renderer/src/components/analyses/analysis-editor.tsx +++ b/flowsint-app/src/renderer/src/components/analyses/analysis-editor.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect } from "react" +import { useState, useEffect, useCallback } from "react" import { useMutation, useQueryClient } from "@tanstack/react-query" import { MinimalTiptapEditor } from "@/components/analyses/editor" import { analysisService } from "@/api/analysis-service" @@ -72,6 +72,29 @@ export const AnalysisEditor = ({ const [editor, setEditor] = useState(undefined) const [isEditingTitle, setIsEditingTitle] = useState(false) + // Debounced save function + const debouncedSave = useCallback( + debounce(() => { + if (analysis && editorValue) { + saveMutation.mutate({}) + } + }, 1000), // 1 second delay + [analysis, editorValue] + ) + + // Debounce function + function debounce(func: Function, wait: number) { + let timeout: NodeJS.Timeout + return function executedFunction(...args: any[]) { + const later = () => { + clearTimeout(timeout) + func(...args) + } + clearTimeout(timeout) + timeout = setTimeout(later, wait) + } + } + // Mutations const createMutation = useMutation({ mutationFn: async () => { @@ -103,7 +126,11 @@ export const AnalysisEditor = ({ })) }, onSuccess: async (data) => { - queryClient.invalidateQueries({ queryKey: ["analyses", investigationId] }) + // Use more specific query invalidation + queryClient.setQueryData(["analyses", investigationId], (oldData: Analysis[] | undefined) => { + if (!oldData) return oldData + return oldData.map(item => item.id === data.id ? data : item) + }) onAnalysisUpdate?.(data) toast.success("Analysis saved !") }, @@ -135,7 +162,11 @@ export const AnalysisEditor = ({ })) }, onSuccess: async (data) => { - queryClient.invalidateQueries({ queryKey: ["analyses", investigationId] }) + // Use more specific query invalidation + queryClient.setQueryData(["analyses", investigationId], (oldData: Analysis[] | undefined) => { + if (!oldData) return oldData + return oldData.map(item => item.id === data.id ? data : item) + }) onAnalysisUpdate?.(data) toast.success("Title updated") }, @@ -213,10 +244,10 @@ export const AnalysisEditor = ({ } return ( -
+
{/* Header */} {showHeader && ( -
+
{/* Left section with navigation and title */}
diff --git a/flowsint-app/src/renderer/src/components/analyses/analysis-page.tsx b/flowsint-app/src/renderer/src/components/analyses/analysis-page.tsx index 2818875..868453c 100644 --- a/flowsint-app/src/renderer/src/components/analyses/analysis-page.tsx +++ b/flowsint-app/src/renderer/src/components/analyses/analysis-page.tsx @@ -1,13 +1,16 @@ import { useParams } from "@tanstack/react-router" import type { Analysis } from "@/types" -import { Skeleton } from "@/components/ui/skeleton" import { AnalysisEditor } from "./analysis-editor" +import AnalysisSkeleton from "./analysis-skeleton" export const AnalysisPage = ({ analysis, isLoading, isError, refetch }: { analysis: Analysis, isLoading: boolean, isError: boolean, refetch: () => void }) => { const { investigationId } = useParams({ strict: false }) as { investigationId: string, id: string } if (isLoading) { - return + return ( +
+ +
) } if (isError) { diff --git a/flowsint-app/src/renderer/src/components/analyses/analysis-skeleton.tsx b/flowsint-app/src/renderer/src/components/analyses/analysis-skeleton.tsx new file mode 100644 index 0000000..9585916 --- /dev/null +++ b/flowsint-app/src/renderer/src/components/analyses/analysis-skeleton.tsx @@ -0,0 +1,22 @@ +import { Skeleton } from '../ui/skeleton' + +const AnalysisSkeleton = () => { + return ( +
+ + + + + + + +
+ + + + +
+ ) +} + +export default AnalysisSkeleton \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/analyses/editor/minimal-tiptap.tsx b/flowsint-app/src/renderer/src/components/analyses/editor/minimal-tiptap.tsx index 8128589..8980bb7 100644 --- a/flowsint-app/src/renderer/src/components/analyses/editor/minimal-tiptap.tsx +++ b/flowsint-app/src/renderer/src/components/analyses/editor/minimal-tiptap.tsx @@ -99,7 +99,7 @@ export const MinimalTiptapEditor = React.forwardRef< {/* */} diff --git a/flowsint-app/src/renderer/src/components/analyses/notes-panel.tsx b/flowsint-app/src/renderer/src/components/analyses/notes-panel.tsx index 68a3452..5da8008 100644 --- a/flowsint-app/src/renderer/src/components/analyses/notes-panel.tsx +++ b/flowsint-app/src/renderer/src/components/analyses/notes-panel.tsx @@ -2,10 +2,10 @@ import { useQuery } from "@tanstack/react-query" import { useParams } from "@tanstack/react-router" import { analysisService } from "@/api/analysis-service" import type { Analysis } from "@/types" -import { Skeleton } from "@/components/ui/skeleton" import { useAnalysisPanelStore } from "@/stores/analysis-panel-store" import { AnalysisEditor } from "./analysis-editor" import { useEffect } from "react" +import AnalysisSkeleton from "./analysis-skeleton" export const AnalysisPanel = () => { const { investigationId } = useParams({ strict: false }) as { investigationId: string } @@ -53,7 +53,7 @@ export const AnalysisPanel = () => { } if (isLoadingAnalyses) { - return + return } if (isError) { diff --git a/flowsint-app/src/renderer/src/components/chat/chat-history.tsx b/flowsint-app/src/renderer/src/components/chat/chat-history.tsx index f3b7467..bbcca2b 100644 --- a/flowsint-app/src/renderer/src/components/chat/chat-history.tsx +++ b/flowsint-app/src/renderer/src/components/chat/chat-history.tsx @@ -2,7 +2,7 @@ import { chatCRUDService } from '@/api/chat-service'; import { useQuery } from '@tanstack/react-query'; import { SkeletonList } from '../shared/skeleton-list'; import { Button } from '../ui/button'; -import { ArrowLeft, Trash, MessageSquare, Sparkles } from 'lucide-react'; +import { ArrowLeft, Trash, Sparkles } from 'lucide-react'; import { Chat } from '@/types'; import { useCallback } from 'react'; import { formatDistanceToNow } from 'date-fns'; diff --git a/flowsint-app/src/renderer/src/components/chat/chat-prompt.tsx b/flowsint-app/src/renderer/src/components/chat/chat-prompt.tsx index 19e9cb6..f67b6b8 100644 --- a/flowsint-app/src/renderer/src/components/chat/chat-prompt.tsx +++ b/flowsint-app/src/renderer/src/components/chat/chat-prompt.tsx @@ -134,7 +134,7 @@ export const ChatPanel = ({ export const ContextList = memo(({ context }: { context: any }) => { const colors = useNodesDisplaySettings(s => s.colors) return ( -
+
{context.map((item: any, index: number) => { const color = colors[item?.data?.type || item?.type] return ( diff --git a/flowsint-app/src/renderer/src/components/chat/floating-chat.tsx b/flowsint-app/src/renderer/src/components/chat/floating-chat.tsx index e8992eb..aa44508 100644 --- a/flowsint-app/src/renderer/src/components/chat/floating-chat.tsx +++ b/flowsint-app/src/renderer/src/components/chat/floating-chat.tsx @@ -164,7 +164,7 @@ function FloatingChat() {
{/* Content */} -
+
{isLoading ? ( ) : hasMessages ? ( diff --git a/flowsint-app/src/renderer/src/components/chat/memoized-markdown.tsx b/flowsint-app/src/renderer/src/components/chat/memoized-markdown.tsx index 6e84e4b..3acc382 100644 --- a/flowsint-app/src/renderer/src/components/chat/memoized-markdown.tsx +++ b/flowsint-app/src/renderer/src/components/chat/memoized-markdown.tsx @@ -25,7 +25,7 @@ const MemoizedMarkdownBlock = memo( style={dracula} language={match[1]} PreTag="div" - className="rounded border border-border bg-muted px-1 py-0.5 text-sm" + className="rounded border border-border bg-muted px-1 py-0.5 !m-0 text-sm w-auto not-prose" {...props} />
@@ -33,13 +33,13 @@ const MemoizedMarkdownBlock = memo(
) : ( - + {children} ) } }} - remarkPlugins={[remarkGfm]}>{content}; + remarkPlugins={[remarkGfm]}>{content} }, (prevProps, nextProps) => { if (prevProps.content !== nextProps.content) return false; @@ -53,9 +53,11 @@ export const MemoizedMarkdown = memo( ({ content, id }: { content: string; id: string }) => { const blocks = useMemo(() => parseMarkdownIntoBlocks(content), [content]); - return blocks.map((block, index) => ( - - )); + return ( +
{blocks.map((block, index) => ( + + ))} +
) }, ); diff --git a/flowsint-app/src/renderer/src/components/dashboard/active-malware-chart.tsx b/flowsint-app/src/renderer/src/components/dashboard/active-malware-chart.tsx deleted file mode 100644 index 9d54c8e..0000000 --- a/flowsint-app/src/renderer/src/components/dashboard/active-malware-chart.tsx +++ /dev/null @@ -1,77 +0,0 @@ -import { Card } from '@/components/ui/card'; -import { ChartContainer, ChartTooltip, ChartTooltipContent } from '@/components/ui/chart'; -import { PieChart, Pie, Cell, ResponsiveContainer } from 'recharts'; - -const chartData = [ - { name: 'Beacon', value: 24.4, color: 'var(--chart-1)' }, - { name: 'Mirai', value: 17.6, color: 'var(--chart-2)' }, - { name: 'Android', value: 10.7, color: 'var(--chart-3)' }, - { name: 'Trojan', value: 10.2, color: 'var(--chart-4)' }, - { name: 'RAT', value: 8.1, color: 'var(--chart-5)' }, - { name: 'Mozi', value: 6.8, color: 'var(--primary)' }, - { name: 'APT29', value: 6.5, color: 'var(--accent)' }, - { name: 'Emotet', value: 6.3, color: 'var(--muted-foreground)' }, - { name: 'Linux', value: 4.9, color: 'var(--secondary)' }, - { name: 'Qakbot', value: 4.5, color: 'var(--ring)' }, -]; - -const chartConfig = { - value: { - label: 'Percentage', - }, -}; - -export function ActiveMalwareChart() { - return ( - -
-

Active Malware

-
- - - - {chartData.map((entry, index) => ( - - ))} - - { - if (active && payload && payload.length) { - const data = payload[0]; - return ( -
-

{data.payload.name}

-

- {data.value}% -

-
- ); - } - return null; - }} - /> - - -
- {chartData.map((item, index) => ( -
-
- {item.name} - {item.value}% -
- ))} -
- - ); -} \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/dashboard/analysis-cards.tsx b/flowsint-app/src/renderer/src/components/dashboard/analysis-cards.tsx new file mode 100644 index 0000000..4d16f58 --- /dev/null +++ b/flowsint-app/src/renderer/src/components/dashboard/analysis-cards.tsx @@ -0,0 +1,172 @@ +import { useQuery } from "@tanstack/react-query"; +import { analysisService } from "@/api/analysis-service"; +import { Link } from "@tanstack/react-router"; +import { Card, CardContent } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Skeleton } from "@/components/ui/skeleton"; +import { FileText, Plus, Clock } from "lucide-react"; +import { formatDistanceToNow } from "date-fns"; + +interface AnalysisCardProps { + analysis: any; +} + +function AnalysisCard({ analysis }: AnalysisCardProps) { + return ( + + + + {/* Header with title and type */} +
+
+
+ +
+
+

+ {analysis.title} +

+

+ Analysis document +

+
+
+ + Doc + +
+ + {/* Description */} +
+

+ {analysis.description || "No description available"} +

+
+ + {/* Footer with timestamp */} +
+
+ + + Updated {formatDistanceToNow(new Date(analysis.last_updated_at), { addSuffix: true })} + +
+
+ + + + ); +} + +function AnalysisCardsSkeleton() { + return ( +
+
+
+ + +
+ +
+
+ {Array.from({ length: 8 }).map((_, i) => ( + + +
+
+ +
+ + +
+
+ +
+
+ + +
+
+ +
+ + + ))} +
+
+ ); +} + +export function AnalysisCards() { + const { data: analyses, isLoading } = useQuery({ + queryKey: ["analyses", "dashboard"], + queryFn: analysisService.get, + }); + + if (isLoading) { + return ; + } + + if (!analyses || analyses.length === 0) { + return ( +
+
+
+ +

Recent Analyses

+
+ +
+
+ +

No analyses yet

+

+ Create your first analysis to start documenting your findings. +

+ +
+
+ ); + } + + return ( +
+
+
+ +

Recent Analyses

+ + {analyses.length} analyses + +
+
+ +
+
+ +
+ {analyses.slice(0, 8).map((analysis: any) => ( + + ))} +
+
+ ); +} \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/dashboard/discover-section.tsx b/flowsint-app/src/renderer/src/components/dashboard/discover-section.tsx deleted file mode 100644 index 8f866ec..0000000 --- a/flowsint-app/src/renderer/src/components/dashboard/discover-section.tsx +++ /dev/null @@ -1,34 +0,0 @@ -import { PlusCircle, Import, FileText } from 'lucide-react'; -import { Card, CardHeader, CardTitle, CardContent } from '@/components/ui/card'; - -export function DiscoverSection() { - return ( -
-

Discover what you can do

-

Explore the full potential of the platform with these quick actions.

-
- - -
- Start a new investigation - - Quickly create a new investigation to begin your work. - - - -
- Import data - - Import investigations or data from files to get started quickly. - - - -
- Read documentation - - Learn how to use the platform and best practices. - -
-
- ); -} \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/dashboard/get-started-section.tsx b/flowsint-app/src/renderer/src/components/dashboard/get-started-section.tsx deleted file mode 100644 index a7a7091..0000000 --- a/flowsint-app/src/renderer/src/components/dashboard/get-started-section.tsx +++ /dev/null @@ -1,34 +0,0 @@ -import { PlusCircle, Import } from 'lucide-react'; -import NewInvestigation from '@/components/investigations/new-investigation'; - -export function GetStartedSection() { - return ( -
-

Get started

-
- {/* Action 1 */} - -
-
- -
-
- Start a new investigation - Quickly create and organize a new investigation. -
-
- - {/* Action 2 */} -
-
- -
-
- Import data - Easily import your existing investigations or data files. -
-
-
-
- ); -} \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/dashboard/investigation-analyses.tsx b/flowsint-app/src/renderer/src/components/dashboard/investigation-analyses.tsx new file mode 100644 index 0000000..7d162aa --- /dev/null +++ b/flowsint-app/src/renderer/src/components/dashboard/investigation-analyses.tsx @@ -0,0 +1,176 @@ +import { useQuery } from "@tanstack/react-query"; +import { analysisService } from "@/api/analysis-service"; +import { Link } from "@tanstack/react-router"; +import { Card, CardContent } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Skeleton } from "@/components/ui/skeleton"; +import { FileText, Plus, Clock } from "lucide-react"; +import { formatDistanceToNow } from "date-fns"; + +interface InvestigationAnalysesProps { + investigationId: string; +} + +function AnalysisCard({ analysis, investigationId }: { analysis: any; investigationId: string }) { + return ( + + + + {/* Header with title and type */} +
+
+
+ +
+
+

+ {analysis.title} +

+

+ Analysis document +

+
+
+ + Doc + +
+ + {/* Description */} +
+

+ {analysis.description || "No description available"} +

+
+ + {/* Footer with timestamp */} +
+
+ + + Updated {formatDistanceToNow(new Date(analysis.last_updated_at), { addSuffix: true })} + +
+
+ + + + ); +} + +function InvestigationAnalysesSkeleton() { + return ( +
+
+
+ + +
+ +
+
+ {Array.from({ length: 8 }).map((_, i) => ( + + +
+
+ +
+ + +
+
+ +
+
+ + +
+
+ +
+ + + ))} +
+
+ ); +} + +export function InvestigationAnalyses({ investigationId }: InvestigationAnalysesProps) { + const { data: analyses, isLoading } = useQuery({ + queryKey: ["analyses", "investigation", investigationId], + queryFn: () => analysisService.getByInvestigationId(investigationId), + staleTime: 30000, // 30 seconds + refetchOnWindowFocus: false, + refetchOnMount: false, + }); + + if (isLoading) { + return ; + } + + if (!analyses || analyses.length === 0) { + return ( +
+
+
+ +

Analyses

+
+ +
+
+ +

No analyses yet

+

+ Create your first analysis to start documenting your findings. +

+ +
+
+ ); + } + + return ( +
+
+
+ +

Analyses

+ + {analyses.length} analyses + +
+
+ + +
+
+ +
+ {analyses.slice(0, 8).map((analysis: any) => ( + + ))} +
+
+ ); +} \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/dashboard/investigation-cards.tsx b/flowsint-app/src/renderer/src/components/dashboard/investigation-cards.tsx new file mode 100644 index 0000000..f05ed06 --- /dev/null +++ b/flowsint-app/src/renderer/src/components/dashboard/investigation-cards.tsx @@ -0,0 +1,187 @@ +import { useQuery } from "@tanstack/react-query"; +import { investigationService } from "@/api/investigation-service"; +import { Link } from "@tanstack/react-router"; +import { Card, CardContent } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Skeleton } from "@/components/ui/skeleton"; +import { FolderOpen, Plus, Clock, FileText, BarChart3 } from "lucide-react"; +import { formatDistanceToNow } from "date-fns"; +import NewInvestigation from "@/components/investigations/new-investigation"; + +interface InvestigationCardProps { + investigation: any; +} + +function InvestigationCard({ investigation }: InvestigationCardProps) { + const sketchCount = investigation.sketches?.length || 0; + const analysisCount = investigation.analyses?.length || 0; + const totalItems = sketchCount + analysisCount; + + return ( + + + + {/* Header with title and count */} +
+
+
+ +
+
+

+ {investigation.name} +

+

+ {totalItems} items +

+
+
+ + {totalItems} + +
+ + {/* Stats row */} +
+
+ + {sketchCount} graphs +
+
+ + {analysisCount} docs +
+
+ + {/* Footer with timestamp */} +
+
+ + + Updated {formatDistanceToNow(new Date(investigation.last_updated_at), { addSuffix: true })} + +
+
+ + + + ); +} + +function InvestigationCardsSkeleton() { + return ( +
+
+
+ + +
+ +
+
+ {Array.from({ length: 8 }).map((_, i) => ( + + +
+
+ +
+ + +
+
+ +
+
+ + +
+
+ +
+ + + ))} +
+
+ ); +} + +export function InvestigationCards() { + const { data: investigations, isLoading } = useQuery({ + queryKey: ["investigations", "dashboard"], + queryFn: investigationService.get, + staleTime: 30000, // 30 seconds + refetchOnWindowFocus: false, + refetchOnMount: false, + }); + + if (isLoading) { + return ; + } + + if (!investigations || investigations.length === 0) { + return ( +
+
+
+ +

Investigations

+
+ + + +
+
+ +

No investigations yet

+

+ Create your first investigation to start organizing your data. +

+ + + +
+
+ ); + } + + return ( +
+
+
+ +

Investigations

+ + {investigations.length} investigations + +
+
+ + + +
+
+ +
+ {investigations.slice(0, 8).map((investigation: any) => ( + + ))} +
+
+ ); +} \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/dashboard/investigation-skeleton.tsx b/flowsint-app/src/renderer/src/components/dashboard/investigation-skeleton.tsx index 241df65..2f805c8 100644 --- a/flowsint-app/src/renderer/src/components/dashboard/investigation-skeleton.tsx +++ b/flowsint-app/src/renderer/src/components/dashboard/investigation-skeleton.tsx @@ -4,14 +4,14 @@ import { Card, CardContent, CardHeader } from '@/components/ui/card' export function InvestigationSkeleton() { return (
-
+
{/* Recent Investigations Skeleton */}
-
+
{[...Array(2)].map((_, i) => (
@@ -34,7 +34,7 @@ export function InvestigationSkeleton() { {/* Get Started Section Skeleton */}
-
+
{[...Array(3)].map((_, i) => ( @@ -53,7 +53,7 @@ export function InvestigationSkeleton() { {/* Discover Section Skeleton */}
-
+
{[...Array(2)].map((_, i) => ( @@ -72,7 +72,7 @@ export function InvestigationSkeleton() { {/* Useful Resources Skeleton */}
-
+
{[...Array(2)].map((_, i) => ( diff --git a/flowsint-app/src/renderer/src/components/dashboard/investigation-sketches.tsx b/flowsint-app/src/renderer/src/components/dashboard/investigation-sketches.tsx new file mode 100644 index 0000000..a951851 --- /dev/null +++ b/flowsint-app/src/renderer/src/components/dashboard/investigation-sketches.tsx @@ -0,0 +1,186 @@ +import { useQuery } from "@tanstack/react-query"; +import { investigationService } from "@/api/investigation-service"; +import { Link } from "@tanstack/react-router"; +import { Card, CardContent } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Skeleton } from "@/components/ui/skeleton"; +import { BarChart3, Plus, Clock } from "lucide-react"; +import { formatDistanceToNow } from "date-fns"; +import NewSketch from "@/components/graphs/new-sketch"; + +interface InvestigationSketchesProps { + investigationId: string; + title: string +} + +function SketchCard({ sketch, investigationId }: { sketch: any; investigationId: string }) { + return ( + + + + {/* Header with title and type */} +
+
+
+ +
+
+

+ {sketch.title} +

+

+ Graph sketch +

+
+
+ + Graph + +
+ + {/* Description */} +
+

+ {sketch.description || "No description available"} +

+
+ + {/* Footer with timestamp */} +
+
+ + + Updated {formatDistanceToNow(new Date(sketch.last_updated_at), { addSuffix: true })} + +
+
+ + + + ); +} + +function InvestigationSketchesSkeleton() { + return ( +
+
+
+ + +
+ +
+
+ {Array.from({ length: 8 }).map((_, i) => ( + + +
+
+ +
+ + +
+
+ +
+
+ + +
+
+ +
+ + + ))} +
+
+ ); +} + +export function InvestigationSketches({ investigationId, title }: InvestigationSketchesProps) { + const { data: investigation, isLoading } = useQuery({ + queryKey: ["investigation", "sketches", investigationId], + queryFn: () => investigationService.getById(investigationId), + staleTime: 30000, // 30 seconds + refetchOnWindowFocus: false, + refetchOnMount: false, + }); + + if (isLoading) { + return ; + } + + const sketches = investigation?.sketches || []; + + if (sketches.length === 0) { + return ( +
+
+
+ +

Sketches

+
+ + + +
+
+ +

No sketches yet

+

+ Create your first sketch to start visualizing your investigation data. +

+ + + +
+
+ ); + } + + return ( +
+
+
+ +

{title}

+ + {sketches.length} sketches + +
+
+ + + + +
+
+ +
+ {sketches.slice(0, 8).map((sketch: any) => ( + + ))} +
+
+ ); +} \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/dashboard/metrics-card.tsx b/flowsint-app/src/renderer/src/components/dashboard/metrics-card.tsx deleted file mode 100644 index 5e1f672..0000000 --- a/flowsint-app/src/renderer/src/components/dashboard/metrics-card.tsx +++ /dev/null @@ -1,57 +0,0 @@ -import { Card } from '@/components/ui/card'; -import { TrendingUp, TrendingDown, Database, Network, FileText } from 'lucide-react'; -import { cn } from '@/lib/utils'; - -interface MetricsCardProps { - title: string; - value: string; - trend: { - value: number; - period: string; - isPositive: boolean; - }; - type: 'entities' | 'relationships' | 'reports'; -} - -const iconMap = { - entities: Database, - relationships: Network, - reports: FileText, -}; - -const colorMap = { - entities: 'text-chart-1', - relationships: 'text-chart-2', - reports: 'text-chart-3', -}; - -export function MetricsCard({ title, value, trend, type }: MetricsCardProps) { - const Icon = iconMap[type]; - const TrendIcon = trend.isPositive ? TrendingUp : TrendingDown; - - return ( - -
-
-
- -
-
-

{title}

-

{value}

-
-
-
-
- - {trend.value.toLocaleString()} -
-

{trend.period}

-
-
- - ); -} \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/dashboard/targeted-sectors-chart.tsx b/flowsint-app/src/renderer/src/components/dashboard/targeted-sectors-chart.tsx deleted file mode 100644 index 59ce3e1..0000000 --- a/flowsint-app/src/renderer/src/components/dashboard/targeted-sectors-chart.tsx +++ /dev/null @@ -1,114 +0,0 @@ -import { Card } from '@/components/ui/card'; -import { ChartContainer, ChartTooltip, ChartTooltipContent } from '@/components/ui/chart'; -import { LineChart, Line, XAxis, YAxis, ResponsiveContainer, Legend } from 'recharts'; - -const chartData = [ - { date: 'Sep 7, 2023', government: 40, finance: 30, manufacturing: 25, telecommunications: 35, defense: 20 }, - { date: 'Sep 14, 2023', government: 80, finance: 65, manufacturing: 45, telecommunications: 55, defense: 40 }, - { date: 'Sep 21, 2023', government: 165, finance: 90, manufacturing: 85, telecommunications: 75, defense: 60 }, - { date: 'Sep 28, 2023', government: 45, finance: 35, manufacturing: 40, telecommunications: 30, defense: 25 }, - { date: 'Oct 5, 2023', government: 85, finance: 55, manufacturing: 60, telecommunications: 45, defense: 35 }, - { date: 'Oct 12, 2023', government: 35, finance: 25, manufacturing: 30, telecommunications: 20, defense: 15 }, -]; - -const chartConfig = { - government: { - label: 'Government and administration', - color: 'var(--chart-1)', - }, - finance: { - label: 'Finance', - color: 'var(--chart-2)', - }, - manufacturing: { - label: 'Manufacturing', - color: 'var(--chart-3)', - }, - telecommunications: { - label: 'Telecommunications', - color: 'var(--chart-4)', - }, - defense: { - label: 'Defense', - color: 'var(--chart-5)', - }, -}; - -export function TargetedSectorsChart() { - return ( - -
-

Targeted Sectors

-
- - - - - } /> - - - - - - - -
- {Object.entries(chartConfig).map(([key, config]) => ( -
-
- {config.label} -
- ))} -
- - ); -} \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/dashboard/useful-resources.tsx b/flowsint-app/src/renderer/src/components/dashboard/useful-resources.tsx deleted file mode 100644 index 558360a..0000000 --- a/flowsint-app/src/renderer/src/components/dashboard/useful-resources.tsx +++ /dev/null @@ -1,41 +0,0 @@ -import { FileText, Users } from 'lucide-react'; -import { Button } from '@/components/ui/button'; -import { Card, CardHeader, CardTitle, CardDescription, CardContent } from '@/components/ui/card'; - -export function UsefulResources() { - return ( -
-

Useful Resources

-
- - - API Reference - Explore the API for integrations and automation. - - - - - - - - Community & Support - Get help or connect with other users. - - - - - -
-
- ); -} \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/graphs/wall/custom/context-menu.tsx b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx similarity index 97% rename from flowsint-app/src/renderer/src/components/graphs/wall/custom/context-menu.tsx rename to flowsint-app/src/renderer/src/components/graphs/context-menu.tsx index 46af4b1..993ee05 100644 --- a/flowsint-app/src/renderer/src/components/graphs/wall/custom/context-menu.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/context-menu.tsx @@ -10,15 +10,17 @@ import { GraphNode } from '@/stores/graph-store'; import { useLaunchTransform } from '@/hooks/use-launch-transform'; import { useParams } from '@tanstack/react-router'; import { capitalizeFirstLetter, cn } from '@/lib/utils'; -import NodeActions from '../../node-actions'; +import NodeActions from '@/components/graphs/node-actions'; import BaseContextMenu from '@/components/xyflow/context-menu'; interface GraphContextMenuProps { node: GraphNode; - top: number; - left: number; - right: number; - bottom: number; + top?: number; + left?: number; + right?: number; + bottom?: number; + rawTop?: number; + rawLeft?: number; wrapperWidth: number; wrapperHeight: number; onEdit?: () => void; diff --git a/flowsint-app/src/renderer/src/components/graphs/create-relation.tsx b/flowsint-app/src/renderer/src/components/graphs/create-relation.tsx index e8da95f..ee4debb 100644 --- a/flowsint-app/src/renderer/src/components/graphs/create-relation.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/create-relation.tsx @@ -25,7 +25,7 @@ export function CreateRelationDialog() { const newEdge = { source: selectedNodes[0], target: selectedNodes[1], - type: direction, + type: "custom", label: relationType, sketch_id: sketchId } diff --git a/flowsint-app/src/renderer/src/components/graphs/details-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/details-panel.tsx index d66a81d..7e0ae78 100644 --- a/flowsint-app/src/renderer/src/components/graphs/details-panel.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/details-panel.tsx @@ -1,19 +1,39 @@ import { memo } from "react" import { cn } from "@/lib/utils" import { CopyButton } from "@/components/copy" -import { Check, Rocket, X } from "lucide-react" +import { Check, Rocket, X, MousePointer } from "lucide-react" import LaunchTransform from "./launch-transform" import NodeActions from "./node-actions" import { GraphNode } from "@/stores/graph-store" import { Button } from "../ui/button" import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "../ui/tooltip" +import { useIcon } from "@/hooks/use-icon" +import { useParams } from "@tanstack/react-router" +import NeighborsGraph from "./neighbors" -export default function DetailsPanel({ node }: { node: GraphNode }) { +export default function DetailsPanel({ node }: { node: GraphNode | null }) { + const { id: sketchId } = useParams({ strict: false }) + if (!node) { + return ( +
+
+ +
+

+ No Node Selected +

+

+ Click on any node in the graph to view its details and properties +

+
+ ) + } return ( -
-
-

{node.data?.label}

+
+ +
+

{node.data?.label}

@@ -48,6 +68,9 @@ export default function DetailsPanel({ node }: { node: GraphNode }) {
)} +
+ +
) } @@ -101,3 +124,16 @@ function KeyValueDisplay({ data, className }: KeyValueDisplayProps) { } export const MemoizedKeyValueDisplay = memo(KeyValueDisplay) + +const ItemHero = ({ node }: { node: GraphNode }) => { + const ItemIcon = useIcon(node.data.type, node.data.src) + return ( +
+
+

{node.data.label}

+

{node.data.type}

+
+ +
+ ) +} \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/graphs/draggable-item.tsx b/flowsint-app/src/renderer/src/components/graphs/draggable-item.tsx index 3bfc87b..59c66c4 100644 --- a/flowsint-app/src/renderer/src/components/graphs/draggable-item.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/draggable-item.tsx @@ -68,8 +68,8 @@ export const DraggableItem = memo(function DraggableItem({ )} style={{ borderLeftColor: colorStr }} > -
- +
+

{label}

diff --git a/flowsint-app/src/renderer/src/components/graphs/dynamic-form.tsx b/flowsint-app/src/renderer/src/components/graphs/dynamic-form.tsx index ce9eae8..6d718dc 100644 --- a/flowsint-app/src/renderer/src/components/graphs/dynamic-form.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/dynamic-form.tsx @@ -40,7 +40,7 @@ export function DynamicForm({ currentNodeType, initialData = {}, onSubmit, isFor schemaMap[field.name] = field.required ? z .string() - .min(1, { message: `${field.label} est requis` }) + .min(1, { message: `${field.label} is required` }) .email({ message: "Email invalide" }) : z.string().email({ message: "Email invalide" }).optional() break @@ -48,7 +48,7 @@ export function DynamicForm({ currentNodeType, initialData = {}, onSubmit, isFor schemaMap[field.name] = field.required ? z .string() - .min(1, { message: `${field.label} est requis` }) + .min(1, { message: `${field.label} is required` }) .url({ message: "URL invalide" }) : z.string().url({ message: "URL invalide" }).optional() break @@ -56,7 +56,7 @@ export function DynamicForm({ currentNodeType, initialData = {}, onSubmit, isFor schemaMap[field.name] = field.required ? z .string() - .min(1, { message: `${field.label} est requis` }) + .min(1, { message: `${field.label} is required` }) .refine((val) => !isNaN(Number(val)), { message: "Doit être un nombre valide", }) @@ -73,7 +73,7 @@ export function DynamicForm({ currentNodeType, initialData = {}, onSubmit, isFor schemaMap[field.name] = field.required ? z .string() - .min(1, { message: `${field.label} est requis` }) + .min(1, { message: `${field.label} is required` }) .refine((val) => validValues.includes(val), { message: "Valeur non valide", }) @@ -85,13 +85,13 @@ export function DynamicForm({ currentNodeType, initialData = {}, onSubmit, isFor .optional() } else { schemaMap[field.name] = field.required - ? z.string().min(1, { message: `${field.label} est requis` }) + ? z.string().min(1, { message: `${field.label} is required` }) : z.string().optional() } break case "date": schemaMap[field.name] = field.required - ? z.string().min(1, { message: `${field.label} est requis` }) + ? z.string().min(1, { message: `${field.label} is required` }) : z.string().optional() break case "hidden": @@ -99,19 +99,19 @@ export function DynamicForm({ currentNodeType, initialData = {}, onSubmit, isFor break case "textarea": schemaMap[field.name] = field.required - ? z.string().min(1, { message: `${field.label} est requis` }) + ? z.string().min(1, { message: `${field.label} is required` }) : z.string().optional() break case "metadata": schemaMap[field.name] = field.required ? z.record(z.string(), z.string()).refine((val) => Object.keys(val).length > 0, { - message: "Au moins une métadonnée est requise", + message: "Au moins une métadonnée is requirede", }) : z.record(z.string(), z.string()).optional() break default: schemaMap[field.name] = field.required - ? z.string().min(1, { message: `${field.label} est requis` }) + ? z.string().min(1, { message: `${field.label} is required` }) : z.string().optional() } }) diff --git a/flowsint-app/src/renderer/src/components/graphs/empty-state.tsx b/flowsint-app/src/renderer/src/components/graphs/empty-state.tsx index 91faf1c..200101d 100644 --- a/flowsint-app/src/renderer/src/components/graphs/empty-state.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/empty-state.tsx @@ -78,7 +78,7 @@ const EmptyState = memo(() => { className="h-10 px-6 font-medium shadow-lg hover:shadow-xl transition-all duration-200 hover:scale-105" size="default" > - + Add your first item

diff --git a/flowsint-app/src/renderer/src/components/graphs/force-controls.tsx b/flowsint-app/src/renderer/src/components/graphs/force-controls.tsx new file mode 100644 index 0000000..b3feb81 --- /dev/null +++ b/flowsint-app/src/renderer/src/components/graphs/force-controls.tsx @@ -0,0 +1,100 @@ +import { useGraphSettingsStore } from '@/stores/graph-settings-store'; +import { Popover, PopoverContent, PopoverTrigger } from '../ui/popover'; +import { Slider } from '../ui/slider'; +import { Label } from '../ui/label'; +import { Zap, Move, Target } from 'lucide-react'; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '../ui/select'; +import { useCallback } from 'react'; + +const ForceControls = ({ children }: { children: React.ReactNode }) => { + const settings = useGraphSettingsStore(s => s.settings); + const currentPreset = useGraphSettingsStore(s => s.currentPreset); + const updateSetting = useGraphSettingsStore(s => s.updateSetting); + const applyPreset = useGraphSettingsStore(s => s.applyPreset); + const getPresets = useGraphSettingsStore(s => s.getPresets); + const setSettingsModalOpen = useGraphSettingsStore(s => s.setSettingsModalOpen) + + const handleOpenSettingsModal = useCallback(() => { + setSettingsModalOpen(true) + }, [setSettingsModalOpen]) + + const quickControls = [ + { + key: 'chargeStrength', + label: 'Repulsion', + icon: , + }, + { + key: 'linkStrength', + label: 'Link Force', + icon: , + }, + { + key: 'd3VelocityDecay', + label: 'Friction', + icon: , + } + ]; + + const presets = getPresets(); + + return ( + + +

+ {children} +
+ + +
+
Force Presets
+ + +
+
Quick Controls
+ {quickControls.map(({ key, label, icon }) => { + const setting = settings[key]; + if (!setting) return null; + + return ( +
+
+ {icon} + +
+ updateSetting(key, values[0])} + min={setting.min} + max={setting.max} + step={setting.step} + className="w-full" + /> +
+ ); + })} +
+ +
+ +
+
+ + + ); +}; + +export default ForceControls; \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/graphs/force-graph-viewer.tsx b/flowsint-app/src/renderer/src/components/graphs/force-graph-viewer.tsx deleted file mode 100644 index 778f707..0000000 --- a/flowsint-app/src/renderer/src/components/graphs/force-graph-viewer.tsx +++ /dev/null @@ -1,289 +0,0 @@ -import { ItemType, useNodesDisplaySettings } from '@/stores/node-display-settings'; -import React, { useCallback, useMemo, useEffect, useState, useRef } from 'react'; -import ForceGraph2D from 'react-force-graph-2d'; - -export interface GraphNode { - id: string; - data?: { - type?: string; - label?: string; - }; - label?: string; - position?: { x: number; y: number }; -} - -export interface GraphEdge { - id: string; - source: string; - target: string; - label?: string; -} - -interface ForceGraphViewerProps { - nodes: GraphNode[]; - edges: GraphEdge[]; - width?: number; - height?: number; - nodeColors?: Record; - nodeSizes?: Record; - onNodeClick?: (node: GraphNode) => void; - onNodeRightClick?: (node: GraphNode, event: MouseEvent) => void; - onBackgroundClick?: () => void; - showLabels?: boolean; - showIcons?: boolean; - backgroundColor?: string; - className?: string; - style?: React.CSSProperties; - onGraphRef?: (ref: any) => void; -} - -const NODE_COUNT_THRESHOLD = 500; - -const ForceGraphViewer: React.FC = ({ - nodes, - edges, - width, - height, - onNodeClick, - onNodeRightClick, - onBackgroundClick, - showLabels = true, - showIcons = true, - backgroundColor = 'transparent', - className = '', - style, - onGraphRef -}) => { - const [currentZoom, setCurrentZoom] = useState(1); - const [dimensions, setDimensions] = useState({ width: 0, height: 0 }); - const nodeColors = useNodesDisplaySettings(s => s.colors) - const getSize = useNodesDisplaySettings(s => s.getSize); - // Call onGraphRef when graphRef changes - useEffect(() => { - if (onGraphRef && graphRef.current) { - onGraphRef(graphRef.current); - } - }, [onGraphRef]); - const containerRef = useRef(null); - const graphRef = useRef(); - - const shouldUseSimpleRendering = useMemo(() => { - return nodes.length > NODE_COUNT_THRESHOLD || currentZoom < 2.5; - }, [nodes.length, currentZoom]); - - // Transform data for Force Graph - const graphData = useMemo(() => { - const transformedNodes = nodes.map(node => { - const type = node.data?.type as ItemType; - const color = nodeColors[type] || '#0074D9'; - const size = getSize(type); - let nodeLabel = node.data?.label || node.label || node.id; - - return { - ...node, - nodeLabel: nodeLabel, - nodeColor: color, - nodeSize: size, - nodeType: type, - val: size, - }; - }); - - const edgeGroups = new Map(); - edges.forEach(edge => { - const key = `${edge.source}-${edge.target}`; - if (!edgeGroups.has(key)) { - edgeGroups.set(key, []); - } - edgeGroups.get(key).push(edge); - }); - - const transformedEdges = edges.map((edge) => { - const key = `${edge.source}-${edge.target}`; - const group = edgeGroups.get(key); - const groupIndex = group.indexOf(edge); - const groupSize = group.length; - const curve = groupSize > 1 ? (groupIndex - (groupSize - 1) / 2) * 0.2 : 0; - - return { - ...edge, - edgeLabel: edge.label, - curve: curve, - groupIndex: groupIndex, - groupSize: groupSize - }; - }); - - return { - nodes: transformedNodes, - links: transformedEdges, - }; - }, [nodes, edges, nodeColors, getSize]); - - const handleNodeClick = useCallback((node: any) => { - if (onNodeClick) { - onNodeClick(node); - } - }, [onNodeClick]); - - const handleNodeRightClick = useCallback((node: any, event: MouseEvent) => { - if (onNodeRightClick) { - onNodeRightClick(node, event); - } - }, [onNodeRightClick]); - - const handleBackgroundClick = useCallback(() => { - if (onBackgroundClick) { - onBackgroundClick(); - } - }, [onBackgroundClick]); - - const renderNode = useCallback((node: any, ctx: CanvasRenderingContext2D, globalScale: number) => { - const size = node.nodeSize; - const type = node.nodeType; - - if (shouldUseSimpleRendering) { - // Simple circle rendering for large node counts - ctx.beginPath(); - ctx.arc(node.x, node.y, size * 0.65, 0, 2 * Math.PI); - ctx.fillStyle = node.nodeColor; - ctx.fill(); - } else { - // Full rendering with icon and label - ctx.beginPath(); - ctx.fillStyle = node.nodeColor; - ctx.fill(); - - if (showIcons) { - const img = new Image(); - img.src = `/icons/${type}.svg`; - ctx.drawImage(img, node.x - size / 2, node.y - size / 2, size, size); - } - - if (showLabels && globalScale > 3) { - const label = node.nodeLabel || node.label || node.id; - if (label) { - const fontSize = Math.max(2, size * 0.4); - ctx.font = `${fontSize}px -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif`; - const textWidth = ctx.measureText(label).width; - const padding = 2; - - ctx.fillStyle = 'rgba(255, 255, 255, 0.45)'; - ctx.strokeStyle = 'rgba(0, 0, 0, 0.05)'; - ctx.lineWidth = 0.2; - const bgWidth = textWidth + padding * 2; - const bgHeight = fontSize + padding; - const bgX = node.x - bgWidth / 2; - const bgY = node.y + size / 2 + 1; - - ctx.beginPath(); - ctx.roundRect(bgX, bgY, bgWidth, bgHeight, 0.75); - ctx.fill(); - ctx.stroke(); - - ctx.fillStyle = 'rgb(0, 0, 0)'; - ctx.textAlign = 'center'; - ctx.textBaseline = 'middle'; - ctx.fillText(label, node.x, bgY + bgHeight / 2); - } - } - } - }, [shouldUseSimpleRendering, showLabels, showIcons]); - - // Handle container resize - useEffect(() => { - if (!containerRef.current) return; - - const resizeObserver = new ResizeObserver(entries => { - const { width: w, height: h } = entries[0].contentRect; - setDimensions({ width: w, height: h }); - if (graphRef.current) { - graphRef.current.d3ReheatSimulation(); - } - }); - - resizeObserver.observe(containerRef.current); - - return () => { - resizeObserver.disconnect(); - }; - }, []); - - if (!nodes.length) { - return ( -
-
-

No nodes to display

-
-
- ); - } - - return ( -
- shouldUseSimpleRendering ? node.nodeColor : "#00000000"} - nodeRelSize={6} - linkDirectionalArrowLength={3.5} - linkDirectionalArrowRelPos={1} - cooldownTicks={100} - onNodeRightClick={handleNodeRightClick} - onNodeClick={handleNodeClick} - onBackgroundClick={handleBackgroundClick} - linkCurvature={link => link.curve} - linkDirectionalParticles={link => link.__highlighted ? 2 : 0} - linkDirectionalParticleSpeed={0.005} - linkWidth={link => link.__highlighted ? 2 : link.__dimmed ? 0.5 : 1} - linkColor={link => { - if (link.__highlighted) return 'rgba(171, 171, 171, 0.23)'; - if (link.__dimmed) return 'rgba(146, 146, 146, 0.33)'; - return 'rgba(166, 166, 166, 0.32)'; - }} - d3AlphaDecay={0.02} - d3VelocityDecay={0.3} - nodeCanvasObject={renderNode} - onNodeHover={(node: any) => { - if (node) { - graphData.nodes.forEach((n: any) => { - n.__highlighted = n === node; - n.__dimmed = n !== node; - }); - graphData.links.forEach((l: any) => { - l.__highlighted = l.source === node || l.target === node; - l.__dimmed = l.source !== node && l.target !== node; - }); - } else { - graphData.nodes.forEach((n: any) => { - n.__highlighted = false; - n.__dimmed = false; - }); - graphData.links.forEach((l: any) => { - l.__highlighted = false; - l.__dimmed = false; - }); - } - }} - backgroundColor={backgroundColor} - onZoom={(zoom) => setCurrentZoom(zoom.k)} - /> -
- ); -}; - -export default ForceGraphViewer; \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx new file mode 100644 index 0000000..f0364b7 --- /dev/null +++ b/flowsint-app/src/renderer/src/components/graphs/graph-main.tsx @@ -0,0 +1,76 @@ +import { useGraphStore } from '@/stores/graph-store' +import React, { useEffect, useRef, useCallback } from 'react' +import GraphViewer from './graph-viewer' +import ContextMenu from './context-menu' + +const GraphMain = () => { + const nodes = useGraphStore(s => s.nodes) + const edges = useGraphStore(s => s.edges) + // const currentNode = useGraphStore(s => s.currentNode) + const toggleNodeSelection = useGraphStore(s => s.toggleNodeSelection) + const clearSelectedNodes = useGraphStore(s => s.clearSelectedNodes) + + const graphRef = useRef() + const containerRef = useRef(null) + const [menu, setMenu] = React.useState(null) + + // // Handle current node centering + // useEffect(() => { + // if (!currentNode || !graphRef.current) return + // graphRef.current.centerAt(currentNode.x, currentNode.y, 1000) + // graphRef.current.zoom(5, 2000) + // }, [currentNode]) + + const handleNodeClick = useCallback((node: any) => { + toggleNodeSelection(node, false) + }, [toggleNodeSelection]) + + const handleBackgroundClick = useCallback(() => { + clearSelectedNodes() + setMenu(null) + }, [clearSelectedNodes]) + + const onNodeContextMenu = useCallback((node: any, event: MouseEvent) => { + if (!containerRef.current || !node) return + + const pane = containerRef.current.getBoundingClientRect() + const relativeX = event.clientX - pane.left + const relativeY = event.clientY - pane.top + + setMenu({ + node: { + data: node.data || {}, + id: node.id || '', + label: node.label || node.nodeLabel || '', + position: node.position || { x: node.x || 0, y: node.y || 0 } + }, + rawTop: relativeY, + rawLeft: relativeX, + wrapperWidth: pane.width, + wrapperHeight: pane.height, + setMenu: setMenu, + }) + }, []) + + const handleGraphRef = useCallback((ref: any) => { + graphRef.current = ref + }, []) + + return ( +
+ + {menu && } +
+ ) +} + +export default GraphMain \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-navigation.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-navigation.tsx index 5a9c9ab..f6417ff 100644 --- a/flowsint-app/src/renderer/src/components/graphs/graph-navigation.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/graph-navigation.tsx @@ -3,16 +3,16 @@ import NodesPanel from "./nodes-panel" import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" import { UserPlus, Users } from "lucide-react" import { ItemsPanel } from "./items-panel" -import DetailsPanel from "./details-panel" import { ResizableHandle, ResizablePanel, ResizablePanelGroup } from "../ui/resizable" import { useLayoutStore } from "@/stores/layout-store" +import SelectedItemsPanel from "./selected-items-panel" const GraphNavigation = () => { const nodes = useGraphStore((s) => s.nodes) - const currentNode = useGraphStore((s) => s.currentNode) const activeTab = useLayoutStore((s) => s.activeTab) const setActiveTab = useLayoutStore((s) => s.setActiveTab) - + const selectedNodes = useGraphStore(s => s.selectedNodes) + const selectedNodesSome = selectedNodes.length > 0 return (
{ direction="vertical" className="flex-1 min-h-0 flex w-full flex-col" > - {currentNode && ( + {selectedNodesSome && ( <> { defaultSize={30} className="flex flex-col w-full overflow-hidden min-h-0 min-w-0" > - + diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx index 57204a9..692b2d2 100644 --- a/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/graph-panel.tsx @@ -7,7 +7,6 @@ import { ArrowDownToLineIcon } from 'lucide-react' import { CreateRelationDialog } from './create-relation' import GraphLoader from './graph-loader' import Loader from '../loader' -// import WallEditor from './wall/wall' import { useGraphControls } from '@/stores/graph-controls-store' import { NodeEditorModal } from './node-editor-modal' import NodesTable from '../table' @@ -16,13 +15,13 @@ import { useActionItems } from '@/hooks/use-action-items' import { toast } from 'sonner' import MapPanel from '../map/map-panel' import NewActions from './new-actions' -const GraphReactForce = lazy(() => import('./graph-react-force')) +import GraphSettings from './graph-settings' +import GraphMain from './graph-main' const RelationshipsTable = lazy(() => import('@/components/table/relationships-view')) -// const GraphReactSigma = lazy(() => import('./graph-react-sigma')) +const Graph = lazy(() => import('./graph')) +const Wall = lazy(() => import('./wall/wall')) -// const Graph = lazy(() => import('./graph')) - -const NODE_COUNT_THRESHOLD = 1000; +const NODE_COUNT_THRESHOLD = 2000; // Separate component for the drag overlay const DragOverlay = memo(({ isDragging }: { isDragging: boolean }) => ( @@ -45,9 +44,8 @@ interface GraphPanelProps { isRefetching: boolean; } -const GraphPanel = ({ graphData, isLoading, isRefetching }: GraphPanelProps) => { +const GraphPanel = ({ graphData, isLoading }: GraphPanelProps) => { const graphPanelRef = useRef(null) - const [loading, setLoading] = useState(isLoading) const handleOpenFormModal = useGraphStore(s => s.handleOpenFormModal) const nodes = useGraphStore(s => s.nodes) const view = useGraphControls((s) => s.view) @@ -62,9 +60,8 @@ const GraphPanel = ({ graphData, isLoading, isRefetching }: GraphPanelProps) => updateGraphData([], []) if (graphData?.nds && graphData?.rls) { updateGraphData(graphData.nds, graphData.rls) - setLoading(false) } - }, [graphData?.nds, graphData?.rls, setLoading]) + }, [graphData?.nds, graphData?.rls]) const handleDragOver = (e: React.DragEvent) => { e.preventDefault() @@ -134,20 +131,21 @@ const GraphPanel = ({ graphData, isLoading, isRefetching }: GraphPanelProps) => }> {nodes?.length > NODE_COUNT_THRESHOLD ? ( <>{view === "table" && } - {["force", "hierarchy"].includes(view) && } + {["force", "hierarchy"].includes(view) && } {view === "map" && } {view === "relationships" && } - ) : (<> - {view === "force" && } - {/* {view === "hierarchy" && } */} - {view === "table" && } - {view === "map" && } - {view === "relationships" && } - )} + ) : ( + <> + {view === "hierarchy" && } + {view === "force" && } + {/* {["force", "hierarchy"].includes(view) && } */} + {view === "table" && } + {view === "map" && } + {view === "relationships" && } + + )} - {/* */} - {/* */}
@@ -155,6 +153,7 @@ const GraphPanel = ({ graphData, isLoading, isRefetching }: GraphPanelProps) => +
) } diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-react-force-3d.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-react-force-3d.tsx deleted file mode 100644 index aaef335..0000000 --- a/flowsint-app/src/renderer/src/components/graphs/graph-react-force-3d.tsx +++ /dev/null @@ -1,186 +0,0 @@ -import React, { useCallback, useEffect, useState, useMemo, useRef } from 'react'; -import ForceGraph3D from 'react-force-graph-3d'; -import { GraphEdge, GraphNode, useGraphStore } from '@/stores/graph-store'; -import { useNodesDisplaySettings } from '@/stores/node-display-settings'; -import { useGraphControls } from '@/stores/graph-controls-store'; -import type { ItemType } from '@/stores/node-display-settings'; -import EmptyState from './empty-state'; -//@ts-ignore -import SpriteText from "https://esm.sh/three-spritetext"; - -interface GraphReactForce3DProps { - style?: React.CSSProperties; -} - -const GraphReactForce3D: React.FC = () => { - const nodes = useGraphStore(s => s.nodes) as GraphNode[]; - const edges = useGraphStore(s => s.edges) as GraphEdge[]; - const colors = useNodesDisplaySettings(s => s.colors) as Record; - const toggleNodeSelection = useGraphStore(s => s.toggleNodeSelection); - const clearSelectedNodes = useGraphStore(s => s.clearSelectedNodes); - const setActions = useGraphControls(s => s.setActions); - const getSize = useNodesDisplaySettings(s => s.getSize); - const graphRef = useRef(); - const containerRef = useRef(null); - const [dimensions, setDimensions] = useState({ width: 0, height: 0 }); - const currentNode = useGraphStore(s => s.currentNode); - - // Cache pour les sprites de texte des liens - const linkSpritesCache = useRef>(new Map()); - - // Mémoriser les données du graphique pour éviter les re-rendus - const graphData = useMemo(() => ({ - nodes, - links: edges - }), [nodes, edges]); - - // Mémoriser les fonctions de couleur et de taille pour éviter les recalculs - const nodeColorFunction = useCallback((node: GraphNode) => { - return colors[node.data.type]; - }, [colors]); - - const nodeValueFunction = useCallback((node: GraphNode) => { - return getSize(node.data.type as ItemType) / 4; - }, [getSize]); - - // Optimisation des sprites de texte des liens avec cache - const linkThreeObjectFunction = useCallback((link: GraphEdge) => { - const cacheKey = `${link.source}-${link.target}-${link.label}`; - - if (linkSpritesCache.current.has(cacheKey)) { - return linkSpritesCache.current.get(cacheKey); - } - - const sprite = new SpriteText(`${link.label}`); - sprite.color = 'lightgrey'; - sprite.textHeight = 1.5; - - linkSpritesCache.current.set(cacheKey, sprite); - return sprite; - }, []); - - // Nettoyer le cache quand les liens changent - useEffect(() => { - linkSpritesCache.current.clear(); - }, [edges]); - - const handleNodeClick = useCallback((node: any) => { - toggleNodeSelection(node, false); - const distance = 130; - const distRatio = 1 + distance / Math.hypot(node.x, node.y, node.z); - - graphRef.current?.cameraPosition( - { x: node.x * distRatio, y: node.y * distRatio, z: node.z * distRatio }, - node, - 500 - ); - }, [toggleNodeSelection]); - - useEffect(() => { - if (currentNode) { - handleNodeClick(currentNode); - } - }, [currentNode, handleNodeClick]); - - const handleBackgroundClick = useCallback(() => { - clearSelectedNodes(); - }, [clearSelectedNodes]); - - const handleZoomIn = useCallback((graph: any) => { - const distance = graph.cameraDistance(); - graph.cameraDistance(distance * 0.75); - }, []); - - const handleZoomOut = useCallback((graph: any) => { - const distance = graph.cameraDistance(); - graph.cameraDistance(distance * 1.25); - }, []); - - const handleFit = useCallback((graph: any) => { - graph.zoomToFit(400); - }, []); - - // Set actions in store - useEffect(() => { - setActions({ - zoomIn: () => handleZoomIn(graphRef.current), - zoomOut: () => handleZoomOut(graphRef.current), - zoomToFit: () => handleFit(graphRef.current), - }); - }, [setActions, handleZoomIn, handleZoomOut, handleFit]); - - // Handle container resize avec debouncing - useEffect(() => { - if (!containerRef.current) return; - - let timeoutId: NodeJS.Timeout; - const resizeObserver = new ResizeObserver(entries => { - clearTimeout(timeoutId); - timeoutId = setTimeout(() => { - const { width, height } = entries[0].contentRect; - setDimensions({ width, height }); - if (graphRef.current) { - graphRef.current.d3ReheatSimulation(); - } - }, 100); // Debounce de 100ms - }); - - resizeObserver.observe(containerRef.current); - - return () => { - clearTimeout(timeoutId); - resizeObserver.disconnect(); - }; - }, []); - - if (!nodes.length) { - return ; - } - - return ( -
-
- link.curve} - backgroundColor="#00000000" - enableNodeDrag={true} - enableNavigationControls={true} - // Optimiser les sprites de texte - linkThreeObjectExtend={true} - linkThreeObject={linkThreeObjectFunction} - showNavInfo={false} - // Limiter la distance de rendu pour de meilleures performances - nodeOpacity={0.9} - linkOpacity={0.6} - /> -
-
- ); -}; - -export default GraphReactForce3D; \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-react-force.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-react-force.tsx deleted file mode 100644 index 1a9edce..0000000 --- a/flowsint-app/src/renderer/src/components/graphs/graph-react-force.tsx +++ /dev/null @@ -1,486 +0,0 @@ -import React, { useCallback, useMemo, useEffect, useState } from 'react'; -import ForceGraph2D from 'react-force-graph-2d'; -import { GraphEdge, GraphNode, useGraphStore } from '@/stores/graph-store'; -import { useNodesDisplaySettings } from '@/stores/node-display-settings'; -import { useGraphControls } from '@/stores/graph-controls-store'; -import type { ItemType } from '@/stores/node-display-settings'; -import EmptyState from './empty-state'; -import { useTheme } from '../theme-provider'; -import ContextMenu from './wall/custom/context-menu'; - -interface GraphReactForceProps { - style?: React.CSSProperties; -} - -interface LabelBounds { - x: number; - y: number; - width: number; - height: number; - nodeId: string; - nodeSize: number; -} - -const NODE_COUNT_THRESHOLD = 1000; -const ZOOM_MIN = 0.3; -const ZOOM_INTERVAL = 2; -const ZOOM_MAX = 10; -const ZOOM_THRESHOLD = 4; - -const GraphReactForce: React.FC = () => { - const nodes = useGraphStore(s => s.nodes) as GraphNode[]; - const rawEdges = useGraphStore(s => s.edges) as GraphEdge[]; - const { theme } = useTheme(); - const getSize = useNodesDisplaySettings(s => s.getSize); - const colors = useNodesDisplaySettings(s => s.colors) as Record; - const toggleNodeSelection = useGraphStore(s => s.toggleNodeSelection); - const clearSelectedNodes = useGraphStore(s => s.clearSelectedNodes); - const setActions = useGraphControls(s => s.setActions); - const [menu, setMenu] = useState(null); - const [currentZoom, setCurrentZoom] = useState(1); - - const shouldUseSimpleRendering = useMemo(() => { - return nodes.length > NODE_COUNT_THRESHOLD || currentZoom < ZOOM_THRESHOLD; - }, [nodes.length, currentZoom]); - - // Transform data for Force Graph - const graphData = useMemo(() => { - const transformedNodes = nodes.map(node => { - const type = node.data?.type as ItemType; - const color = colors[type] || '#0074D9'; - const size = getSize(type)-2; - let nodeLabel = node.data.label; - if (!nodeLabel && 'caption' in node) nodeLabel = (node as any).caption; - if (!nodeLabel) nodeLabel = node.id; - const randMin = ZOOM_MIN + Math.random() * (ZOOM_MAX - ZOOM_MIN - 1); - const randMax = randMin + ZOOM_INTERVAL; - return { - ...node, - nodeLabel: nodeLabel, - nodeColor: color, - nodeSize: size, - nodeType: type, - val: size, - randMin: randMin, - randMax: randMax - }; - }).sort((a, b) => b.nodeSize - a.nodeSize); // Sort by size, largest first - - const edgeGroups = new Map(); - rawEdges.forEach(edge => { - const key = `${edge.source}-${edge.target}`; - if (!edgeGroups.has(key)) { - edgeGroups.set(key, []); - } - edgeGroups.get(key).push(edge); - }); - - const transformedEdges = rawEdges.map((edge) => { - const key = `${edge.source}-${edge.target}`; - const group = edgeGroups.get(key); - const groupIndex = group.indexOf(edge); - const groupSize = group.length; - const curve = groupSize > 1 ? (groupIndex - (groupSize - 1) / 2) * 0.2 : 0; - - return { - ...edge, - edgeLabel: edge.label, - curve: curve, - groupIndex: groupIndex, - groupSize: groupSize - }; - }); - - return { - nodes: transformedNodes, - links: transformedEdges, - }; - }, [nodes, rawEdges, getSize, colors]); - - const handleNodeClick = useCallback((node: any) => { - toggleNodeSelection(node, false); - }, [toggleNodeSelection]); - - const handleBackgroundClick = useCallback(() => { - clearSelectedNodes(); - setMenu(null) - }, [clearSelectedNodes, setMenu]); - - const handleZoomIn = useCallback((graph: any) => { - const zoom = graph.zoom(); - graph.zoom(zoom * 1.5); - }, []); - - const handleZoomOut = useCallback((graph: any) => { - const zoom = graph.zoom(); - graph.zoom(zoom * 0.75); - }, []); - - const handleFit = useCallback((graph: any) => { - graph.zoomToFit(400); - }, []); - - // Helper functions for rendering - const drawNodeCircle = useCallback((ctx: CanvasRenderingContext2D, node: any, size: number) => { - ctx.beginPath(); - ctx.arc(node.x, node.y, size * 0.65, 0, 2 * Math.PI); - ctx.fillStyle = node.nodeColor; - ctx.fill(); - - // Add border with same color - ctx.strokeStyle = node.nodeColor; - ctx.lineWidth = 0.75; - ctx.stroke(); - }, []); - - const drawNodeIcon = useCallback((ctx: CanvasRenderingContext2D, node: any, size: number, type: ItemType) => { - // Draw circular border first - ctx.beginPath(); - ctx.arc(node.x, node.y, size * 0.70, 0, 2 * Math.PI); - ctx.strokeStyle = node.nodeColor; - ctx.lineWidth = 0.75; - ctx.stroke(); - - // Draw icon on top - const img = new Image(); - img.src = `/icons/${type}.svg`; - ctx.drawImage(img, node.x - size / 2, node.y - size / 2, size, size); - }, []); - - const shouldShowLabel = useCallback((globalScale: number, randMin: number, randMax: number) => { - return globalScale >= randMin && globalScale <= randMax; - }, []); - - // Check if two rectangles overlap - const doLabelsOverlap = useCallback((bounds1: LabelBounds, bounds2: LabelBounds) => { - return !(bounds1.x + bounds1.width < bounds2.x || - bounds2.x + bounds2.width < bounds1.x || - bounds1.y + bounds1.height < bounds2.y || - bounds2.y + bounds2.height < bounds1.y); - }, []); - - const drawLabelBackground = useCallback((ctx: CanvasRenderingContext2D, node: any, label: string, fontSize: number) => { - const textWidth = ctx.measureText(label).width; - const padding = 6; - const bgWidth = textWidth + padding * 2; - const bgHeight = fontSize + padding; - const bgX = node.x - bgWidth / 2; - const bgY = node.y + node.nodeSize / 2 + 1; - - ctx.fillStyle = 'rgba(149, 149, 149, 0.56)'; - ctx.strokeStyle = 'rgba(0, 0, 0, 0.05)'; - ctx.lineWidth = 0.2; - - ctx.beginPath(); - ctx.roundRect(bgX, bgY, bgWidth, bgHeight, 4); - ctx.fill(); - ctx.stroke(); - - return { bgX, bgY, bgHeight, bgWidth }; - }, []); - - const drawLabel = useCallback((ctx: CanvasRenderingContext2D, node: any, label: string, fontSize: number, bgY: number, bgHeight: number) => { - ctx.fillStyle = theme === "dark" ? 'rgb(255, 255, 255)' : 'rgb(0, 0, 0)'; - ctx.textAlign = 'center'; - ctx.textBaseline = 'middle'; - ctx.fillText(label, node.x, bgY + bgHeight / 2); - }, [theme]); - - // Efficient collision detection using spatial partitioning - const collisionGridRef = React.useRef>(new Map()); - const renderedLabelsRef = React.useRef>(new Set()); - const gridSize = 50; - - // Get all grid keys that a label bounds might overlap - const getOverlappingGridKeys = useCallback((bounds: LabelBounds) => { - const keys = new Set(); - const startX = Math.floor(bounds.x / gridSize); - const endX = Math.floor((bounds.x + bounds.width) / gridSize); - const startY = Math.floor(bounds.y / gridSize); - const endY = Math.floor((bounds.y + bounds.height) / gridSize); - - for (let x = startX; x <= endX; x++) { - for (let y = startY; y <= endY; y++) { - keys.add(`${x},${y}`); - } - } - return Array.from(keys); - }, []); - - // Check for collisions using spatial partitioning with size-based priority - const checkCollision = useCallback((newBounds: LabelBounds) => { - const gridKeys = getOverlappingGridKeys(newBounds); - - for (const key of gridKeys) { - const boundsInGrid = collisionGridRef.current.get(key) || []; - for (const existingBounds of boundsInGrid) { - if (doLabelsOverlap(newBounds, existingBounds)) { - // If the new label is from a larger node, it takes priority - if (newBounds.nodeSize > existingBounds.nodeSize) { - // Remove the smaller node's label from the grid - const updatedBounds = boundsInGrid.filter(b => b.nodeId !== existingBounds.nodeId); - collisionGridRef.current.set(key, updatedBounds); - renderedLabelsRef.current.delete(existingBounds.nodeId); - continue; // Check other bounds in this grid - } else { - // New label is from a smaller node, so it gets rejected - return true; - } - } - } - } - return false; - }, [getOverlappingGridKeys, doLabelsOverlap]); - - // Add bounds to spatial grid - const addBoundsToGrid = useCallback((bounds: LabelBounds) => { - const gridKeys = getOverlappingGridKeys(bounds); - for (const key of gridKeys) { - if (!collisionGridRef.current.has(key)) { - collisionGridRef.current.set(key, []); - } - collisionGridRef.current.get(key)!.push(bounds); - } - }, [getOverlappingGridKeys]); - - // Custom renderer with efficient collision detection - const renderNodeWithCollisionDetection = useCallback((node: any, ctx: CanvasRenderingContext2D, globalScale: number) => { - const type = node.nodeType as ItemType; - const label = node.nodeLabel || node.label || node.id; - - // Always render the node itself - if (shouldUseSimpleRendering) { - drawNodeCircle(ctx, node, node.nodeSize); - } else { - drawNodeIcon(ctx, node, node.nodeSize, type); - } - - // Check if we should show a label - if (!shouldShowLabel(globalScale, node.randMin, node.randMax)) return; - - // Determine font size based on zoom level - let fontSize: number; - if (shouldUseSimpleRendering) { - fontSize = Math.max(8, 14 / globalScale); - } else { - if (globalScale > 3) { - fontSize = Math.max(2, node.nodeSize * 0.4); - } else { - fontSize = Math.max(8, 14 / globalScale); - } - } - - // Set font for measurement - ctx.font = `${fontSize}px -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif`; - - // Measure text to check for collisions - const textWidth = ctx.measureText(label).width; - const padding = 2; - const bgWidth = textWidth + padding * 2; - const bgHeight = fontSize + padding; - const bgX = node.x - bgWidth / 2; - const bgY = node.y + node.nodeSize / 2 + 1; - - const newLabelBounds: LabelBounds = { - x: bgX, - y: bgY, - width: bgWidth, - height: bgHeight, - nodeId: node.id, - nodeSize: node.nodeSize - }; - - // Check if this label was previously rendered - const wasPreviouslyRendered = renderedLabelsRef.current.has(node.id); - - // If not previously rendered, check for collisions - if (!wasPreviouslyRendered) { - if (checkCollision(newLabelBounds)) { - return; // Don't render this label - } - // Add this label to the spatial grid - addBoundsToGrid(newLabelBounds); - // Mark as rendered - renderedLabelsRef.current.add(node.id); - } - - // Render the label background and text - if (shouldUseSimpleRendering || globalScale > 3) { - // Draw background for detailed labels - const { bgY: finalBgY, bgHeight: finalBgHeight } = drawLabelBackground(ctx, node, label, fontSize); - drawLabel(ctx, node, label, fontSize, finalBgY, finalBgHeight); - } else { - // Draw simple label without background - ctx.fillStyle = theme === "dark" ? 'rgb(255, 255, 255)' : 'rgb(0, 0, 0)'; - ctx.textAlign = 'center'; - ctx.textBaseline = 'middle'; - ctx.fillText(label, node.x, node.y); - } - }, [shouldUseSimpleRendering, drawNodeCircle, drawNodeIcon, shouldShowLabel, checkCollision, addBoundsToGrid, drawLabelBackground, drawLabel, theme]); - - // Reset collision grid and rendered labels only when nodes change - useEffect(() => { - collisionGridRef.current.clear(); - renderedLabelsRef.current.clear(); - }, [nodes.length, nodes.map(n => n.id).join(',')]); // Also reset when node IDs change - - // Reset only when zoom changes significantly - const lastZoomRef = React.useRef(currentZoom); - useEffect(() => { - const zoomDiff = Math.abs(currentZoom - lastZoomRef.current); - if (zoomDiff > 0.5) { - collisionGridRef.current.clear(); - renderedLabelsRef.current.clear(); - lastZoomRef.current = currentZoom; - } - }, [currentZoom]); - - // Set actions in store - useEffect(() => { - setActions({ - zoomIn: () => handleZoomIn(graphRef.current), - zoomOut: () => handleZoomOut(graphRef.current), - zoomToFit: () => handleFit(graphRef.current), - }); - }, [setActions, handleZoomIn, handleZoomOut, handleFit]); - - const graphRef = React.useRef(); - const containerRef = React.useRef(null); - const [dimensions, setDimensions] = useState({ width: 0, height: 0 }); - - // Handle container resize - useEffect(() => { - if (!containerRef.current) return; - - const resizeObserver = new ResizeObserver(entries => { - const { width, height } = entries[0].contentRect; - setDimensions({ width, height }); - if (graphRef.current) { - graphRef.current.d3ReheatSimulation(); - } - }); - - resizeObserver.observe(containerRef.current); - - return () => { - resizeObserver.disconnect(); - }; - }, []); - - const onNodeContextMenu = useCallback( - (data: any, event: MouseEvent) => { - if (!containerRef.current) return; - const pane = containerRef.current.getBoundingClientRect(); - const relativeX = event.clientX - pane.left; - const relativeY = event.clientY - pane.top; - const menuWidth = 320; - const menuHeight = 250; - const padding = 20; - - const wouldOverflowRight = relativeX + menuWidth + padding > pane.width; - const wouldOverflowBottom = relativeY + menuHeight + padding > pane.height; - - let finalTop = 0; - let finalLeft = 0; - let finalRight = 0; - let finalBottom = 0; - - if (wouldOverflowRight) { - finalRight = pane.width - relativeX; - } else { - finalLeft = relativeX; - } - - if (wouldOverflowBottom) { - finalBottom = pane.height - relativeY; - } else { - finalTop = relativeY; - } - - setMenu({ - node: { data: data.data, id: data.id, label: data.label, position: data.position } as GraphNode, - top: finalTop, - left: finalLeft, - right: finalRight, - bottom: finalBottom, - wrapperWidth: pane.width, - wrapperHeight: pane.height, - setMenu: setMenu, - }); - }, - [setMenu], - ); - - if (!nodes.length) { - return ; - } - - return ( -
- {/* {currentZoom} */} -
- shouldUseSimpleRendering ? node.nodeColor : "#00000000"} - nodeRelSize={6} - enableNodeDrag={false} - linkDirectionalArrowLength={3.5} - linkDirectionalArrowRelPos={1} - cooldownTicks={100} - onNodeRightClick={onNodeContextMenu} - onNodeClick={handleNodeClick} - onBackgroundClick={handleBackgroundClick} - linkCurvature={link => link.curve} - linkDirectionalParticles={link => link.__highlighted ? 2 : 0} - linkDirectionalParticleSpeed={0.005} - linkWidth={link => link.__highlighted ? 2 : link.__dimmed ? 0.5 : 1} - linkColor={link => { - if (link.__highlighted) return 'rgba(171, 171, 171, 0.23)'; - if (link.__dimmed) return 'rgba(146, 146, 146, 0.33)'; - return 'rgba(166, 166, 166, 0.32)'; - }} - d3AlphaDecay={0.02} - d3VelocityDecay={0.3} - nodeCanvasObject={renderNodeWithCollisionDetection} - onNodeHover={(node: any) => { - if (node) { - graphData.nodes.forEach((n: any) => { - n.__highlighted = n === node; - n.__dimmed = n !== node; - }); - graphData.links.forEach((l: any) => { - l.__highlighted = l.source === node || l.target === node; - l.__dimmed = l.source !== node && l.target !== node; - }); - } else { - graphData.nodes.forEach((n: any) => { - n.__highlighted = false; - n.__dimmed = false; - }); - graphData.links.forEach((l: any) => { - l.__highlighted = false; - l.__dimmed = false; - }); - } - }} - backgroundColor="transparent" - onZoom={(zoom) => setCurrentZoom(zoom.k)} - /> - - {menu && } -
-
- ); -}; - -export default GraphReactForce; - - - diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-react-sigma.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-react-sigma.tsx deleted file mode 100644 index 0ed82ac..0000000 --- a/flowsint-app/src/renderer/src/components/graphs/graph-react-sigma.tsx +++ /dev/null @@ -1,140 +0,0 @@ -import React, { useEffect, useRef, useMemo } from 'react'; -import Graph from 'graphology'; -import Sigma from 'sigma'; -import { GraphEdge, GraphNode, useGraphStore } from '@/stores/graph-store'; -import { useNodesDisplaySettings } from '@/stores/node-display-settings'; -import { useGraphControls } from '@/stores/graph-controls-store'; -import type { ItemType } from '@/stores/node-display-settings'; - -const GraphReactSigma: React.FC = () => { - const containerRef = useRef(null); - const sigmaRef = useRef(null); - const nodes = useGraphStore(s => s.nodes) as GraphNode[]; - const edges = useGraphStore(s => s.edges) as GraphEdge[]; - const getSize = useNodesDisplaySettings(s => s.getSize); - const colors = useNodesDisplaySettings(s => s.colors) as Record; - const toggleNodeSelection = useGraphStore(s => s.toggleNodeSelection); - const clearSelectedNodes = useGraphStore(s => s.clearSelectedNodes); - const setActions = useGraphControls(s => s.setActions); - - console.log(edges) - // Memoize processed data for performance - const processedNodes = useMemo(() => { - return nodes.map(node => ({ - id: node.id, - label: node.data?.label || node.id, - type: node.data?.type as ItemType, - size: getSize(node.data?.type as ItemType), - color: colors[node.data?.type as ItemType] || '#0074D9', - originalNode: node - })); - }, [nodes, getSize, colors]); - - // Initialize Sigma - useEffect(() => { - if (!containerRef.current) return; - - const graph = new Graph(); - const sigma = new Sigma(graph, containerRef.current, { - defaultNodeType: 'circle', - defaultEdgeType: 'line', - renderEdgeLabels: false, - }); - - sigmaRef.current = sigma; - - // Set up zoom actions - setActions({ - zoomIn: () => sigma.getCamera().animatedZoom({ duration: 300 }), - zoomOut: () => sigma.getCamera().animatedUnzoom({ duration: 300 }), - zoomToFit: () => sigma.getCamera().animatedReset({ duration: 300 }), - }); - - // Set up event handlers - sigma.on('clickNode', (event) => { - const originalNode = processedNodes.find(n => n.id === event.node)?.originalNode; - if (originalNode) { - toggleNodeSelection(originalNode, false); - } - }); - - sigma.on('clickStage', () => { - clearSelectedNodes(); - }); - - return () => { - sigma.kill(); - }; - }, [setActions, toggleNodeSelection, clearSelectedNodes, processedNodes]); - - // Update graph data - useEffect(() => { - if (!sigmaRef.current) return; - if (processedNodes.length === 0) return; - const graph = sigmaRef.current.getGraph(); - graph.clear(); - // Add nodes with simple random positioning - processedNodes.forEach(node => { - const x = (Math.random() - 0.5) * 400; - const y = (Math.random() - 0.5) * 400; - - graph.addNode(node.id, { - label: node.label, - size: node.size, - color: node.color, - x: x, - y: y, - }); - }); - - edges.forEach(edge => { - const sourceExists = graph.hasNode(edge.source); - const targetExists = graph.hasNode(edge.target); - if (sourceExists && targetExists) { - graph.addEdge(edge.source, edge.target, { label: edge.label }); - } - }); - // Force a refresh and fit to view - setTimeout(() => { - sigmaRef.current?.getCamera().animatedReset({ duration: 300 }); - sigmaRef.current?.refresh(); - }, 100); - - }, [processedNodes, edges]); - - // Handle container resize - useEffect(() => { - if (!containerRef.current || !sigmaRef.current) return; - - const resizeObserver = new ResizeObserver(() => { - sigmaRef.current?.refresh(); - }); - - resizeObserver.observe(containerRef.current); - - return () => { - resizeObserver.disconnect(); - }; - }, []); - - if (!nodes.length) { - return ( -
- No nodes to display -
- ); - } - - return ( -
- ); -}; - -export default GraphReactSigma; \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-settings.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-settings.tsx new file mode 100644 index 0000000..d1aaab7 --- /dev/null +++ b/flowsint-app/src/renderer/src/components/graphs/graph-settings.tsx @@ -0,0 +1,125 @@ +import { useGraphSettingsStore } from '@/stores/graph-settings-store' +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { Slider } from "@/components/ui/slider" +import { Label } from '../ui/label' +import { memo, useCallback } from 'react' +import { Button } from '../ui/button' +import { type Setting } from '@/types' +import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '../ui/accordion' + +const GraphSettings = () => { + const settings = useGraphSettingsStore(s => s.settings) + const settingsModalOpen = useGraphSettingsStore(s => s.settingsModalOpen) + const setSettingsModalOpen = useGraphSettingsStore(s => s.setSettingsModalOpen) + const updateSetting = useGraphSettingsStore(s => s.updateSetting) + const resetSettings = useGraphSettingsStore(s => s.resetSettings) + + // Categorize settings + const categories = { + 'Force Physics': [ + 'chargeStrength', + 'linkStrength', + 'centerStrength', + 'collisionRadius', + 'collisionStrength' + ], + 'Simulation Control': [ + 'd3AlphaDecay', + 'd3AlphaMin', + 'd3AlphaTarget', + 'd3VelocityDecay', + 'cooldownTicks', + 'cooldownTime', + 'warmupTicks' + ], + 'Visual Appearance': [ + 'nodeSize', + 'linkWidth', + 'linkDirectionalArrowLength', + 'linkDirectionalArrowRelPos', + 'linkDirectionalParticleSpeed' + ], + 'Layout': [ + 'dagLevelDistance' + ] + } + + return ( + + + + Graph settings + + Update the settings of your graph. Changes apply in real-time. + + + + + {Object.entries(categories).map(([categoryName, settingKeys]) => ( + + + {categoryName} + + + {settingKeys.map((key) => { + const setting = settings[key]; + if (!setting) return null; + return ( + + ); + })} + + + ))} + + +
+ +
+ + + ) +} + +export default GraphSettings + +type SettingItemProps = { + setting: Setting + label: string + updateSetting: (key: string, value: number) => void +} +const SettingItem = memo(({ setting, label, updateSetting }: SettingItemProps) => { + + const handleUpdateSetting = useCallback((newValues: number[]) => { + updateSetting(label, newValues[0]) + }, [setting, updateSetting]) + + return ( +
+ +

{setting.description}

+ +
+ ) +}) \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx new file mode 100644 index 0000000..f71c212 --- /dev/null +++ b/flowsint-app/src/renderer/src/components/graphs/graph-viewer.tsx @@ -0,0 +1,390 @@ +import { getDagreLayoutedElements } from '@/lib/utils'; +import { useGraphControls } from '@/stores/graph-controls-store'; +import { useGraphSettingsStore } from '@/stores/graph-settings-store'; +import { GraphNode, GraphEdge, useGraphStore } from '@/stores/graph-store'; +import { ItemType, useNodesDisplaySettings } from '@/stores/node-display-settings'; +import React, { useCallback, useMemo, useEffect, useState, useRef } from 'react'; +import ForceGraph2D from 'react-force-graph-2d'; +import { Button } from '../ui/button'; +import { useTheme } from '@/components/theme-provider' + +interface GraphViewerProps { + nodes: GraphNode[]; + edges: GraphEdge[]; + width?: number; + height?: number; + nodeColors?: Record; + nodeSizes?: Record; + onNodeClick?: (node: GraphNode) => void; + onNodeRightClick?: (node: GraphNode, event: MouseEvent) => void; + onBackgroundClick?: () => void; + showLabels?: boolean; + showIcons?: boolean; + backgroundColor?: string; + className?: string; + style?: React.CSSProperties; + onGraphRef?: (ref: any) => void; +} + +const NODE_COUNT_THRESHOLD = 500; +const CONSTANTS = { + NODE_DEFAULT_SIZE: 24, + LABEL_FONT_SIZE: 2.5, + NODE_FONT_SIZE: 5, + LABEL_NODE_MARGIN: 18, + PADDING_RATIO: 0.2, + HALF_PI: Math.PI / 2, + PI: Math.PI, + MEASURE_FONT: '1px Sans-Serif', + MIN_FONT_SIZE: 0.5, + LINK_COLOR: 'rgba(128, 128, 128, 0.6)', + LINK_WIDTH: 1, + ARROW_SIZE: 8, + ARROW_ANGLE: Math.PI / 6 +}; + +// Pre-computed constants +const LABEL_FONT_STRING = `${CONSTANTS.LABEL_FONT_SIZE}px Sans-Serif`; + +// Reusable objects to avoid allocations +const tempPos = { x: 0, y: 0 }; +const tempDimensions = [0, 0]; + +const GraphViewer: React.FC = ({ + nodes, + edges, + width, + height, + onNodeClick, + onNodeRightClick, + onBackgroundClick, + showLabels = true, + showIcons = true, + backgroundColor = 'transparent', + className = '', + style, + onGraphRef +}) => { + const [currentZoom, setCurrentZoom] = useState(1); + const [dimensions, setDimensions] = useState({ width: 0, height: 0 }); + + // Store references + const containerRef = useRef(null); + const graphRef = useRef(); + const isGraphReadyRef = useRef(false); + + // Store selectors + const nodeColors = useNodesDisplaySettings(s => s.colors); + const getSize = useNodesDisplaySettings(s => s.getSize); + const settings = useGraphSettingsStore(s => s.settings); + const view = useGraphControls(s => s.view); + const setActions = useGraphControls(s => s.setActions); + const { theme } = useTheme(); + const setOpenMainDialog = useGraphStore(state => state.setOpenMainDialog); + + // Optimized graph initialization callback + const initializeGraph = useCallback((graphInstance: any) => { + if (!graphInstance || isGraphReadyRef.current) return; + isGraphReadyRef.current = true; + // Set up graph actions + setActions({ + zoomIn: () => { + const zoom = graphInstance.zoom(); + graphInstance.zoom(zoom * 1.5); + }, + zoomOut: () => { + const zoom = graphInstance.zoom(); + graphInstance.zoom(zoom * 0.75); + }, + zoomToFit: () => { + graphInstance.zoomToFit(400); + } + }); + + // Call external ref callback + onGraphRef?.(graphInstance); + }, [setActions, onGraphRef]); + + // Handle graph ref changes + useEffect(() => { + if (graphRef.current) { + initializeGraph(graphRef.current); + } + }, [initializeGraph]); + + // Memoized rendering check + const shouldUseSimpleRendering = useMemo(() => + nodes.length > NODE_COUNT_THRESHOLD || currentZoom < 2.5 + , [nodes.length, currentZoom]); + + // Optimized graph data transformation + const graphData = useMemo(() => { + // Transform nodes + const transformedNodes = nodes.map(node => { + const type = node.data?.type as ItemType; + return { + ...node, + nodeLabel: node.data?.label || node.id, + nodeColor: nodeColors[type] || '#0074D9', + nodeSize: CONSTANTS.NODE_DEFAULT_SIZE, + nodeType: type, + val: getSize(type), + }; + }); + + // Group and transform edges + const edgeGroups = new Map(); + edges.forEach(edge => { + const key = `${edge.source}-${edge.target}`; + if (!edgeGroups.has(key)) { + edgeGroups.set(key, []); + } + edgeGroups.get(key)!.push(edge); + }); + + const transformedEdges = edges.map((edge) => { + const key = `${edge.source}-${edge.target}`; + const group = edgeGroups.get(key)!; + const groupIndex = group.indexOf(edge); + const groupSize = group.length; + const curve = groupSize > 1 ? (groupIndex - (groupSize - 1) / 2) * 0.2 : 0; + + return { + ...edge, + edgeLabel: edge.label, + curve, + groupIndex, + groupSize + }; + }); + + // Handle hierarchy layout + if (view === "hierarchy") { + const { nodes: nds, edges: eds } = getDagreLayoutedElements(transformedNodes, transformedEdges); + return { + nodes: nds.map((nd) => ({ ...nd, x: nd.position.x, y: nd.position.y })), + links: eds, + }; + } + + return { + nodes: transformedNodes, + links: transformedEdges, + }; + }, [nodes, edges, nodeColors, getSize, view]); + + // Event handlers + const handleNodeClick = useCallback((node: any) => onNodeClick?.(node), [onNodeClick]); + const handleNodeRightClick = useCallback((node: any, event: MouseEvent) => onNodeRightClick?.(node, event), [onNodeRightClick]); + const handleBackgroundClick = useCallback(() => onBackgroundClick?.(), [onBackgroundClick]); + const handleOpenNewActionDialog = useCallback(() => setOpenMainDialog(true), [setOpenMainDialog]); + + // Optimized node rendering + const renderNode = useCallback((node: any, ctx: CanvasRenderingContext2D, globalScale: number) => { + const size = node.nodeSize * (settings.nodeSize.value / 50); + + // Always draw the basic circle + ctx.beginPath(); + ctx.arc(node.x, node.y, size * 0.65, 0, 2 * Math.PI); + ctx.fillStyle = node.nodeColor; + ctx.fill(); + + // Early exit for simple rendering + if (shouldUseSimpleRendering) return; + + // Icon rendering + if (showIcons) { + const img = new Image(); + img.src = `/icons/${node.nodeType}.svg`; + ctx.drawImage(img, node.x - size / 2, node.y - size / 2, size, size); + } + + // Label rendering + if (showLabels && globalScale > 3) { + const label = node.nodeLabel || node.label || node.id; + if (label) { + const fontSize = CONSTANTS.NODE_FONT_SIZE * (size / 10); + ctx.font = `${fontSize}px Sans-Serif` + const bgHeight = CONSTANTS.NODE_FONT_SIZE + 2; + const bgY = node.y + size / 2 + 1; + + ctx.fillStyle = theme === "light" ? '#161616' : '#FFFFFF'; + ctx.textAlign = 'center'; + ctx.textBaseline = 'middle'; + ctx.fillText(label, node.x, bgY + bgHeight / 2); + } + } + }, [shouldUseSimpleRendering, showLabels, showIcons, settings.nodeSize.value, theme]); + + // Optimized link rendering + const linkCanvasObject = useCallback((link: any, ctx: CanvasRenderingContext2D) => { + const { source: start, target: end } = link; + + // Early exit for unbound links + if (typeof start !== 'object' || typeof end !== 'object') return; + + // Draw connection line + ctx.beginPath(); + ctx.moveTo(start.x, start.y); + ctx.lineTo(end.x, end.y); + ctx.strokeStyle = CONSTANTS.LINK_COLOR; + ctx.lineWidth = CONSTANTS.LINK_WIDTH * (settings.linkWidth.value / 5); + ctx.stroke(); + + // Early exit for simple rendering or no label + if (shouldUseSimpleRendering || !link.label) return; + + // Calculate label position and angle + tempPos.x = (start.x + end.x) * 0.5; + tempPos.y = (start.y + end.y) * 0.5; + + const dx = end.x - start.x; + const dy = end.y - start.y; + let textAngle = Math.atan2(dy, dx); + + // Flip text for readability + if (textAngle > CONSTANTS.HALF_PI || textAngle < -CONSTANTS.HALF_PI) { + textAngle += textAngle > 0 ? -CONSTANTS.PI : CONSTANTS.PI; + } + + // Measure and draw label + ctx.font = LABEL_FONT_STRING; + const textWidth = ctx.measureText(link.label).width; + const padding = CONSTANTS.LABEL_FONT_SIZE * CONSTANTS.PADDING_RATIO; + + tempDimensions[0] = textWidth + padding; + tempDimensions[1] = CONSTANTS.LABEL_FONT_SIZE + padding; + + const halfWidth = tempDimensions[0] * 0.5; + const halfHeight = tempDimensions[1] * 0.5; + + // Batch canvas operations + ctx.save(); + ctx.translate(tempPos.x, tempPos.y); + ctx.rotate(textAngle); + + // Background + ctx.fillStyle = theme === "light" ? "#FFFFFF" : '#161616'; + ctx.fillRect(-halfWidth, -halfHeight, tempDimensions[0], tempDimensions[1]); + + // Text + ctx.fillStyle = 'darkgrey'; + ctx.textAlign = 'center'; + ctx.textBaseline = 'middle'; + ctx.fillText(link.label, 0, 0); + + ctx.restore(); + }, [shouldUseSimpleRendering, settings.linkWidth.value, theme]); + + // Container resize observer + useEffect(() => { + if (!containerRef.current) return; + + const resizeObserver = new ResizeObserver(entries => { + const { width: w, height: h } = entries[0].contentRect; + setDimensions({ width: w, height: h }); + }); + + resizeObserver.observe(containerRef.current); + return () => resizeObserver.disconnect(); + }, []); + + // Restart simulation when settings change + useEffect(() => { + if (graphRef.current && isGraphReadyRef.current) { + graphRef.current.d3ReheatSimulation(); + } + }, [settings]); + + // Empty state + if (!nodes.length) { + return ( +
+
+
+ + + +
+

+ No data to visualize +

+

+ Start your investigation by adding nodes to see them displayed in the graph view. +

+
+

Tip: Use the search bar to find entities or import data to get started

+

Explore: Try searching for domains, emails, or other entities

+
+ +
+
+ ); + } + + return ( +
+ shouldUseSimpleRendering ? node.nodeColor : "#00000000"} + nodeRelSize={6} + onNodeRightClick={handleNodeRightClick} + onNodeClick={handleNodeClick} + onBackgroundClick={handleBackgroundClick} + linkCurvature={link => link.curve} + linkDirectionalParticles={link => link.__highlighted ? 2 : 0} + nodeCanvasObject={renderNode} + onNodeDragEnd={(node => { + node.fx = node.x; + node.fy = node.y; + })} + cooldownTicks={view === "hierarchy" ? 0 : settings.cooldownTicks?.value} + cooldownTime={settings.cooldownTime?.value} + d3AlphaDecay={settings.d3AlphaDecay?.value} + d3AlphaMin={settings.d3AlphaMin?.value} + d3VelocityDecay={settings.d3VelocityDecay?.value} + warmupTicks={settings.warmupTicks?.value} + dagLevelDistance={settings.dagLevelDistance?.value} + linkDirectionalArrowRelPos={settings.linkDirectionalArrowRelPos?.value} + linkDirectionalArrowLength={settings.linkDirectionalArrowLength?.value} + linkDirectionalParticleSpeed={settings.linkDirectionalParticleSpeed?.value} + backgroundColor={backgroundColor} + onZoom={(zoom) => setCurrentZoom(zoom.k)} + linkCanvasObject={linkCanvasObject} + enableNodeDrag={!shouldUseSimpleRendering} + /> +
+ ); +}; + +export default GraphViewer; \ No newline at end of file diff --git a/flowsint-app/src/renderer/src/components/graphs/legend.tsx b/flowsint-app/src/renderer/src/components/graphs/legend.tsx index b76808a..5233197 100644 --- a/flowsint-app/src/renderer/src/components/graphs/legend.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/legend.tsx @@ -19,7 +19,7 @@ const Legend = () => {
- +
    {entries.map(([key, value]) => ( diff --git a/flowsint-app/src/renderer/src/components/graphs/neighbors.tsx b/flowsint-app/src/renderer/src/components/graphs/neighbors.tsx index 57aa10a..0511822 100644 --- a/flowsint-app/src/renderer/src/components/graphs/neighbors.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/neighbors.tsx @@ -1,7 +1,6 @@ import { sketchService } from "@/api/sketch-service"; import { useQuery } from "@tanstack/react-query"; -import { Card, CardContent, CardHeader, CardTitle } from "../ui/card"; -import ForceGraphViewer from "./force-graph-viewer"; +import ForceGraphViewer from "./graph-viewer"; const NeighborsGraph = ({ sketchId, nodeId }: { sketchId: string, nodeId: string }) => { @@ -10,32 +9,23 @@ const NeighborsGraph = ({ sketchId, nodeId }: { sketchId: string, nodeId: string queryFn: () => sketchService.getNodeNeighbors(sketchId, nodeId), }); return ( - - - - Related - - - -
    - {isLoading ?
    Loading...
    :
    -
    {neighborsData.nds.length - 1} neighbor(s)
    - { }} - onNodeRightClick={() => { }} - onBackgroundClick={() => { }} - showLabels={true} - showIcons={true} - backgroundColor="transparent" - onGraphRef={() => { }} - /> -
    } -
    -     -     - +
    + {isLoading ?
    Loading...
    : + <> +
    {neighborsData.nds.length - 1} neighbor(s)
    + { }} + onNodeRightClick={() => { }} + onBackgroundClick={() => { }} + showLabels={true} + showIcons={true} + backgroundColor="transparent" + onGraphRef={() => { }} + /> + } +
    ) } diff --git a/flowsint-app/src/renderer/src/components/graphs/new-actions.tsx b/flowsint-app/src/renderer/src/components/graphs/new-actions.tsx index a2f0b57..59a17ae 100644 --- a/flowsint-app/src/renderer/src/components/graphs/new-actions.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/new-actions.tsx @@ -88,7 +88,7 @@ export default function ActionDialog() { // Add edge to local state if (addEdge) { const newEdgeObject = { - type: "straight", + type: "custom", label: newEdge.label, source: newEdge.source.id, target: newEdge.target.id @@ -146,8 +146,7 @@ export default function ActionDialog() { animate={{ opacity: 1, x: 0 }} exit={{ opacity: 0, x: -20 }} transition={{ duration: 0.2 }} - className="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-3 p-1 pb-2" - > + className="grid grid-cols-3 cq-sm:grid-cols-4 cq-md:grid-cols-5 cq-lg:grid-cols-6 cq-xl:grid-cols-6 gap-3 p-1 pb-2" > {items.map((item) => ( -
    +
    {isLoading ? ( -
    +
    {Array.from({ length: 6 }).map((_, index) => ( diff --git a/flowsint-app/src/renderer/src/components/graphs/node-editor-modal.tsx b/flowsint-app/src/renderer/src/components/graphs/node-editor-modal.tsx index 5a9ca69..1f34e4f 100644 --- a/flowsint-app/src/renderer/src/components/graphs/node-editor-modal.tsx +++ b/flowsint-app/src/renderer/src/components/graphs/node-editor-modal.tsx @@ -5,7 +5,7 @@ import { Input } from "@/components/ui/input" import { Label } from "@/components/ui/label" import { Textarea } from "@/components/ui/textarea" import { Badge } from "@/components/ui/badge" -import { Edit3, Save, X, Hash, Type, FileText, Check, Loader2, User } from "lucide-react" +import { Edit3, Save, X, Hash, Type, FileText, Check, Loader2 } from "lucide-react" import type { NodeData } from "@/types" import { useGraphStore } from "@/stores/graph-store" import { MapFromAddress } from "../map/map" @@ -146,7 +146,7 @@ export const NodeEditorModal: React.FC = () => { case 'core-properties': return (
    -
    +