mirror of
https://github.com/reconurge/flowsint.git
synced 2026-07-11 10:02:31 -05:00
Compare commits
4 Commits
chore/secu
...
fix/prebui
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
82925bec38 | ||
|
|
f38d5d500d | ||
|
|
5cdfe727b4 | ||
|
|
25eb3a0400 |
@@ -5,7 +5,9 @@ MASTER_VAULT_KEY_V1=base64:qnHTmwYb+uoygIw9MsRMY22vS5YPchY+QOi/E79GAvM=
|
||||
NEO4J_URI_BOLT=bolt://neo4j:7687
|
||||
NEO4J_USERNAME=neo4j
|
||||
NEO4J_PASSWORD=password
|
||||
VITE_API_URL=http://127.0.0.1:5001
|
||||
# Dev only (vite dev server / docker-compose.yml). Production images use
|
||||
# same-origin relative URLs proxied by nginx — leave unset for docker-compose.prod.yml.
|
||||
VITE_API_URL=http://localhost:5001
|
||||
# Comma-separated CORS allowed origins. Defaults to http://localhost:5173 (Vite dev) when unset.
|
||||
ALLOWED_ORIGINS=http://localhost:5173
|
||||
DATABASE_URL=postgresql://flowsint:flowsint@localhost:5433/flowsint
|
||||
|
||||
57
Makefile
57
Makefile
@@ -2,18 +2,17 @@ PROJECT_ROOT := $(shell pwd)
|
||||
|
||||
COMPOSE_DEV := docker compose -f docker-compose.dev.yml
|
||||
COMPOSE_PROD := docker compose -f docker-compose.prod.yml
|
||||
COMPOSE_DEPLOY := docker compose -f docker-compose.deploy.yml
|
||||
|
||||
.PHONY: \
|
||||
dev prod deploy \
|
||||
build-dev build-prod \
|
||||
up-dev up-prod up-deploy down \
|
||||
dev prod \
|
||||
build-dev \
|
||||
up-dev up-prod down \
|
||||
infra-dev infra-prod infra-stop-dev infra-stop-prod \
|
||||
migrate-dev migrate-prod \
|
||||
alembic-upgrade alembic-downgrade alembic-revision \
|
||||
api frontend celery \
|
||||
test install clean check-env open-browser-dev open-browser-prod \
|
||||
logs-dev logs-prod logs-deploy status \
|
||||
logs-dev logs-prod status \
|
||||
regenerate-router
|
||||
|
||||
ENV_DIRS := . flowsint-api flowsint-core flowsint-app
|
||||
@@ -63,18 +62,14 @@ open-browser-dev:
|
||||
echo "Frontend ready at http://localhost:5173"
|
||||
|
||||
prod:
|
||||
@echo "Starting PROD environment..."
|
||||
@echo "Starting PROD environment (pre-built images)..."
|
||||
$(MAKE) check-env
|
||||
$(MAKE) build-prod
|
||||
$(COMPOSE_PROD) pull
|
||||
$(MAKE) up-prod
|
||||
@echo ""
|
||||
@echo "Production started!"
|
||||
@echo " Frontend: http://localhost:5173"
|
||||
@echo " API: http://localhost:5001"
|
||||
|
||||
build-prod:
|
||||
@echo "Building PROD images..."
|
||||
$(COMPOSE_PROD) build
|
||||
@echo " API: http://localhost:5173/api (proxied)"
|
||||
|
||||
up-prod:
|
||||
$(COMPOSE_PROD) up -d
|
||||
@@ -91,27 +86,11 @@ logs-prod:
|
||||
$(COMPOSE_PROD) logs -f
|
||||
|
||||
open-browser-prod:
|
||||
@echo "Waiting for frontend on port 80 (Traefik)..."
|
||||
@bash -c 'until curl -s http://localhost > /dev/null 2>&1; do sleep 2; done'
|
||||
@open http://localhost 2>/dev/null || \
|
||||
xdg-open http://localhost 2>/dev/null || \
|
||||
echo "Frontend ready at http://localhost"
|
||||
|
||||
deploy:
|
||||
@echo "Starting DEPLOY environment (GHCR images)..."
|
||||
$(MAKE) check-env
|
||||
$(COMPOSE_DEPLOY) pull
|
||||
$(COMPOSE_DEPLOY) up -d
|
||||
@echo ""
|
||||
@echo "Deploy started!"
|
||||
@echo " Frontend: http://localhost (via Traefik)"
|
||||
@echo " API: http://localhost/api"
|
||||
|
||||
up-deploy:
|
||||
$(COMPOSE_DEPLOY) up -d
|
||||
|
||||
logs-deploy:
|
||||
$(COMPOSE_DEPLOY) logs -f
|
||||
@echo "Waiting for frontend on port 5173..."
|
||||
@bash -c 'until curl -s http://localhost:5173 > /dev/null 2>&1; do sleep 2; done'
|
||||
@open http://localhost:5173 2>/dev/null || \
|
||||
xdg-open http://localhost:5173 2>/dev/null || \
|
||||
echo "Frontend ready at http://localhost:5173"
|
||||
|
||||
migrate-dev:
|
||||
@echo "Running DEV migrations..."
|
||||
@@ -177,7 +156,6 @@ status:
|
||||
down:
|
||||
-$(COMPOSE_DEV) down
|
||||
-$(COMPOSE_PROD) down
|
||||
-$(COMPOSE_DEPLOY) down
|
||||
|
||||
clean:
|
||||
@echo "This will remove ALL Docker data. Continue? [y/N]"
|
||||
@@ -185,7 +163,6 @@ clean:
|
||||
if [ "$$confirm" != "y" ]; then exit 1; fi
|
||||
-$(COMPOSE_DEV) down -v --rmi all --remove-orphans
|
||||
-$(COMPOSE_PROD) down -v --rmi all --remove-orphans
|
||||
-$(COMPOSE_DEPLOY) down -v --rmi all --remove-orphans
|
||||
rm -rf flowsint-app/node_modules
|
||||
rm -rf .venv
|
||||
|
||||
@@ -203,17 +180,11 @@ help:
|
||||
@echo " make logs-dev - Follow DEV logs"
|
||||
@echo " make infra-dev - Start only infra (postgres/redis/neo4j)"
|
||||
@echo ""
|
||||
@echo "Production (local build):"
|
||||
@echo " make prod - Start PROD environment (local build + Traefik)"
|
||||
@echo " make build-prod - Build PROD images"
|
||||
@echo "Production (pre-built GHCR images):"
|
||||
@echo " make prod - Pull images and start PROD environment"
|
||||
@echo " make up-prod - Start PROD containers"
|
||||
@echo " make logs-prod - Follow PROD logs"
|
||||
@echo ""
|
||||
@echo "Deploy (GHCR images):"
|
||||
@echo " make deploy - Start with GHCR images (no build)"
|
||||
@echo " make up-deploy - Start DEPLOY containers"
|
||||
@echo " make logs-deploy - Follow DEPLOY logs"
|
||||
@echo ""
|
||||
@echo "Local (no Docker):"
|
||||
@echo " make api - Run API locally"
|
||||
@echo " make frontend - Run frontend locally"
|
||||
|
||||
40
README.md
40
README.md
@@ -67,12 +67,14 @@ copy .env.example flowsint-core\.env
|
||||
copy .env.example flowsint-app\.env
|
||||
```
|
||||
|
||||
#### 3. Build and start
|
||||
#### 3. Start
|
||||
|
||||
```bat
|
||||
docker compose -f docker-compose.prod.yml up -d --build
|
||||
docker compose -f docker-compose.prod.yml up -d
|
||||
```
|
||||
|
||||
This pulls the pre-built images from GitHub Container Registry — no local build needed.
|
||||
|
||||
### First login
|
||||
|
||||
Then go to [http://localhost:5173/register](http://localhost:5173/register) and create an account. There are no credentials or account by default.
|
||||
@@ -80,6 +82,40 @@ Then go to [http://localhost:5173/register](http://localhost:5173/register) and
|
||||
|
||||
> ✅ OSINT investigations need a high level of privacy. Everything is stored on your machine.
|
||||
|
||||
### Deploy on a network (team / server)
|
||||
|
||||
The same setup works out of the box on a server: the frontend serves the UI **and** proxies all API calls internally, so no extra configuration is needed for clients.
|
||||
|
||||
```bash
|
||||
git clone https://github.com/reconurge/flowsint.git
|
||||
cd flowsint
|
||||
cp .env.example .env
|
||||
# Edit .env — see "Before exposing to a network" below
|
||||
docker compose -f docker-compose.prod.yml up -d
|
||||
```
|
||||
|
||||
Anyone on the network can then access Flowsint at `http://<server-ip>:5173`.
|
||||
|
||||
**Before exposing to a network, change the default secrets in `.env`:**
|
||||
|
||||
- `AUTH_SECRET` — signs authentication tokens. Generate one: `openssl rand -hex 32`
|
||||
- `MASTER_VAULT_KEY_V1` — encrypts stored API keys. Generate one: `python3 -c "import os, base64; print('base64:' + base64.b64encode(os.urandom(32)).decode())"`
|
||||
- `NEO4J_PASSWORD` — Neo4j database password.
|
||||
|
||||
Only port `5173` is exposed to the network. PostgreSQL, Redis, Neo4j and the API are bound to `127.0.0.1` on the server and reachable only through the frontend proxy.
|
||||
|
||||
To pin a specific version instead of `latest`, set `FLOWSINT_VERSION` in `.env` (e.g. `FLOWSINT_VERSION=1.2.10`).
|
||||
|
||||
**HTTPS (recommended beyond a trusted LAN):** put any reverse proxy in front of port 5173. Example with [Caddy](https://caddyserver.com/):
|
||||
|
||||
```
|
||||
flowsint.example.com {
|
||||
reverse_proxy 127.0.0.1:5173
|
||||
}
|
||||
```
|
||||
|
||||
When fronting with a reverse proxy, also bind the app port to localhost in `docker-compose.prod.yml` (`"127.0.0.1:5173:8080"`) so clients can only go through HTTPS.
|
||||
|
||||
|
||||
## What is it?
|
||||
|
||||
|
||||
@@ -10,7 +10,8 @@ services:
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-flowsint}
|
||||
POSTGRES_DB: ${POSTGRES_DB:-flowsint}
|
||||
ports:
|
||||
- "5433:5432"
|
||||
# Bound to localhost: host-only debugging access, never exposed to the network.
|
||||
- "127.0.0.1:5433:5432"
|
||||
volumes:
|
||||
- pg_data_prod:/var/lib/postgresql/data
|
||||
networks:
|
||||
@@ -26,7 +27,7 @@ services:
|
||||
container_name: flowsint-redis-prod
|
||||
restart: always
|
||||
ports:
|
||||
- "6379:6379"
|
||||
- "127.0.0.1:6379:6379"
|
||||
networks:
|
||||
- flowsint_network
|
||||
healthcheck:
|
||||
@@ -40,8 +41,8 @@ services:
|
||||
container_name: flowsint-neo4j-prod
|
||||
restart: always
|
||||
ports:
|
||||
- "7474:7474"
|
||||
- "7687:7687"
|
||||
- "127.0.0.1:7474:7474"
|
||||
- "127.0.0.1:7687:7687"
|
||||
environment:
|
||||
- NEO4J_AUTH=${NEO4J_USERNAME}/${NEO4J_PASSWORD}
|
||||
- NEO4J_PLUGINS=["apoc"]
|
||||
@@ -62,14 +63,13 @@ services:
|
||||
retries: 10
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: flowsint-api/Dockerfile
|
||||
target: production
|
||||
image: ghcr.io/reconurge/flowsint-api:${FLOWSINT_VERSION:-latest}
|
||||
container_name: flowsint-api-prod
|
||||
restart: always
|
||||
ports:
|
||||
- "5001:5001"
|
||||
# The frontend's nginx proxies /api/ to this service over the Docker
|
||||
# network — direct access is host-only, for debugging.
|
||||
- "127.0.0.1:5001:5001"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
environment:
|
||||
@@ -96,10 +96,7 @@ services:
|
||||
- flowsint_network
|
||||
|
||||
celery:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: flowsint-api/Dockerfile
|
||||
target: production
|
||||
image: ghcr.io/reconurge/flowsint-api:${FLOWSINT_VERSION:-latest}
|
||||
container_name: flowsint-celery-prod
|
||||
restart: always
|
||||
command:
|
||||
@@ -144,14 +141,13 @@ services:
|
||||
- flowsint_network
|
||||
|
||||
app:
|
||||
build:
|
||||
context: ./flowsint-app
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
- VITE_API_URL=${VITE_API_URL}
|
||||
# Frontend bundle uses same-origin relative URLs; nginx inside the image
|
||||
# proxies /api/ to the "api" service. No VITE_API_URL needed.
|
||||
image: ghcr.io/reconurge/flowsint-app:${FLOWSINT_VERSION:-latest}
|
||||
container_name: flowsint-app-prod
|
||||
restart: always
|
||||
ports:
|
||||
# The only network-facing port. Serves the UI and proxies /api/ to the API.
|
||||
- "5173:8080"
|
||||
networks:
|
||||
- flowsint_network
|
||||
|
||||
@@ -101,7 +101,7 @@ def is_root_domain(domain: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def is_valid_number(phone: str, region: str = "FR") -> None:
|
||||
def is_valid_number(phone: str, region: str = "FR") -> bool:
|
||||
"""
|
||||
Validates a phone number. Raises InvalidPhoneNumberError if invalid.
|
||||
- `region` should be ISO 3166-1 alpha-2 country code (e.g., 'FR' for France)
|
||||
@@ -113,6 +113,8 @@ def is_valid_number(phone: str, region: str = "FR") -> None:
|
||||
except NumberParseException:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def parse_asn(asn: str) -> int:
|
||||
if not is_valid_asn(asn):
|
||||
|
||||
@@ -10,8 +10,9 @@ RUN yarn install --network-timeout 100000
|
||||
|
||||
COPY . .
|
||||
|
||||
# Build argument for API URL (injected at build time)
|
||||
ARG VITE_API_URL
|
||||
# API URL baked at build time. Empty = same-origin relative URLs (/api/...),
|
||||
# served through the nginx reverse proxy below. Only override for an external API origin.
|
||||
ARG VITE_API_URL=""
|
||||
ENV VITE_API_URL=${VITE_API_URL}
|
||||
|
||||
RUN yarn build
|
||||
|
||||
@@ -57,6 +57,26 @@ http {
|
||||
add_header X-XSS-Protection "1; mode=block" always;
|
||||
add_header Referrer-Policy "strict-origin-when-cross-origin" always;
|
||||
|
||||
# API reverse proxy — same-origin, no CORS needed.
|
||||
# Expects the API service to be reachable as "api:5001" on the Docker network.
|
||||
location /api/ {
|
||||
# Resolve at request time via Docker's embedded DNS: survives api
|
||||
# container restarts and lets nginx boot before api is up.
|
||||
resolver 127.0.0.11 valid=10s;
|
||||
set $api_upstream http://api:5001;
|
||||
proxy_pass $api_upstream;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
# SSE support (event/chat streams)
|
||||
proxy_set_header Connection "";
|
||||
proxy_buffering off;
|
||||
proxy_cache off;
|
||||
proxy_read_timeout 3600s;
|
||||
}
|
||||
|
||||
# Health check endpoint
|
||||
location /health {
|
||||
access_log off;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useAuthStore } from '@/stores/auth-store'
|
||||
|
||||
const API_URL = import.meta.env.VITE_API_URL
|
||||
const API_URL = import.meta.env.VITE_API_URL ?? ''
|
||||
|
||||
export async function fetchWithAuth(endpoint: string, options: RequestInit = {}): Promise<any> {
|
||||
const token = useAuthStore.getState().token
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { DefaultChatTransport } from 'ai'
|
||||
import { useAuthStore } from '@/stores/auth-store'
|
||||
|
||||
const API_URL = import.meta.env.VITE_API_URL
|
||||
const API_URL = import.meta.env.VITE_API_URL ?? ''
|
||||
|
||||
export function createChatTransport(chatId: string) {
|
||||
return new DefaultChatTransport({
|
||||
|
||||
@@ -5,7 +5,7 @@ import { queryKeys } from '@/api/query-keys'
|
||||
import { useAuthStore } from '@/stores/auth-store'
|
||||
import { connectSSE } from '@/api/sse'
|
||||
|
||||
const API_URL = import.meta.env.VITE_API_URL
|
||||
const API_URL = import.meta.env.VITE_API_URL ?? ''
|
||||
|
||||
|
||||
export function useEvents(sketch_id: string | undefined) {
|
||||
|
||||
@@ -4,7 +4,7 @@ import { useAuthStore } from '@/stores/auth-store'
|
||||
import { EventLevel } from '@/types'
|
||||
import { connectSSE } from '@/api/sse'
|
||||
|
||||
const API_URL = import.meta.env.VITE_API_URL
|
||||
const API_URL = import.meta.env.VITE_API_URL ?? ''
|
||||
|
||||
export function useGraphRefresh(sketch_id: string | undefined) {
|
||||
const refetchGraph = useGraphControls((s) => s.refetchGraph)
|
||||
|
||||
@@ -25,7 +25,7 @@ class Vault(VaultProtocol):
|
||||
if not owner_id:
|
||||
raise ValueError("owner_id is required to use the vault.")
|
||||
self.db = db
|
||||
self.owner_id = str(owner_id)
|
||||
self.owner_id = owner_id
|
||||
self.version = "V1"
|
||||
|
||||
def _get_master_key(self) -> bytes:
|
||||
@@ -52,7 +52,7 @@ class Vault(VaultProtocol):
|
||||
algorithm=hashes.SHA256(),
|
||||
length=32,
|
||||
salt=salt,
|
||||
info=self.owner_id.encode("utf-8"),
|
||||
info=str(self.owner_id).encode("utf-8"),
|
||||
)
|
||||
master_key = self._get_master_key()
|
||||
return hkdf.derive(master_key)
|
||||
@@ -72,7 +72,7 @@ class Vault(VaultProtocol):
|
||||
ciphertext = aesgcm.encrypt(
|
||||
iv,
|
||||
plaintext.encode("utf-8"),
|
||||
self.owner_id.encode("utf-8"), # AAD = links secret to user_id
|
||||
str(self.owner_id).encode("utf-8"), # AAD = links secret to user_id
|
||||
)
|
||||
|
||||
return {
|
||||
@@ -92,7 +92,7 @@ class Vault(VaultProtocol):
|
||||
plaintext = aesgcm.decrypt(
|
||||
row.get("iv"),
|
||||
row.get("ciphertext"),
|
||||
self.owner_id.encode("utf-8"),
|
||||
str(self.owner_id).encode("utf-8"),
|
||||
)
|
||||
return plaintext.decode("utf-8")
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ class TestVaultInitialization:
|
||||
"""Test successful Vault initialization."""
|
||||
vault = Vault(db=mock_db, owner_id=owner_id)
|
||||
assert vault.db == mock_db
|
||||
assert vault.owner_id == str(owner_id)
|
||||
assert vault.owner_id == owner_id
|
||||
assert vault.version == "V1"
|
||||
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ class DomainToAsnEnricher(Enricher):
|
||||
vault=vault,
|
||||
params=params,
|
||||
)
|
||||
self.domain_asn_mapping: List[tuple[Domain, ASN]] = []
|
||||
|
||||
@classmethod
|
||||
def required_params(cls) -> bool:
|
||||
@@ -64,6 +65,7 @@ class DomainToAsnEnricher(Enricher):
|
||||
|
||||
async def scan(self, data: List[InputType]) -> List[OutputType]:
|
||||
results: List[OutputType] = []
|
||||
self.domain_asn_mapping = []
|
||||
asnmap = AsnmapTool()
|
||||
|
||||
# Retrieve API key from vault or environment
|
||||
@@ -87,6 +89,7 @@ class DomainToAsnEnricher(Enricher):
|
||||
description=asn_data.get("as_name", ""),
|
||||
)
|
||||
results.append(asn)
|
||||
self.domain_asn_mapping.append((domain, asn))
|
||||
|
||||
Logger.info(
|
||||
self.sketch_id,
|
||||
@@ -115,8 +118,8 @@ class DomainToAsnEnricher(Enricher):
|
||||
self, results: List[OutputType], input_data: List[InputType] = None
|
||||
) -> List[OutputType]:
|
||||
# Create Neo4j relationships between domains and their corresponding ASNs
|
||||
if input_data and self._graph_service:
|
||||
for domain, asn in zip(input_data, results):
|
||||
if self._graph_service:
|
||||
for domain, asn in self.domain_asn_mapping:
|
||||
# Create domain node
|
||||
self.create_node(domain)
|
||||
# Create ASN node
|
||||
|
||||
@@ -14,6 +14,10 @@ class ResolveEnricher(Enricher):
|
||||
InputType = Domain
|
||||
OutputType = Ip
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.domain_ip_mapping: List[tuple[Domain, Ip]] = []
|
||||
|
||||
@classmethod
|
||||
def name(cls) -> str:
|
||||
return "domain_to_ip"
|
||||
@@ -33,10 +37,13 @@ class ResolveEnricher(Enricher):
|
||||
|
||||
async def scan(self, data: List[InputType]) -> List[OutputType]:
|
||||
results: List[OutputType] = []
|
||||
self.domain_ip_mapping = []
|
||||
for d in data:
|
||||
try:
|
||||
ip = socket.gethostbyname(d.domain)
|
||||
results.append(Ip(address=ip))
|
||||
ip_obj = Ip(address=ip)
|
||||
results.append(ip_obj)
|
||||
self.domain_ip_mapping.append((d, ip_obj))
|
||||
except Exception as e:
|
||||
Logger.info(
|
||||
self.sketch_id,
|
||||
@@ -46,7 +53,9 @@ class ResolveEnricher(Enricher):
|
||||
return results
|
||||
|
||||
def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
|
||||
for domain_obj, ip_obj in zip(original_input, results):
|
||||
for domain_obj, ip_obj in self.domain_ip_mapping:
|
||||
if not self._graph_service:
|
||||
continue
|
||||
self.create_node(domain_obj)
|
||||
self.create_node(ip_obj)
|
||||
self.create_relationship(
|
||||
|
||||
@@ -15,6 +15,10 @@ class EmailToGravatarEnricher(Enricher):
|
||||
InputType = Email
|
||||
OutputType = Gravatar
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.email_gravatar_mapping: List[tuple[Email, Gravatar]] = []
|
||||
|
||||
@classmethod
|
||||
def name(cls) -> str:
|
||||
return "email_to_gravatar"
|
||||
@@ -29,6 +33,7 @@ class EmailToGravatarEnricher(Enricher):
|
||||
|
||||
async def scan(self, data: List[InputType]) -> List[OutputType]:
|
||||
results: List[OutputType] = []
|
||||
self.email_gravatar_mapping = []
|
||||
|
||||
for email in data:
|
||||
try:
|
||||
@@ -72,6 +77,7 @@ class EmailToGravatarEnricher(Enricher):
|
||||
|
||||
gravatar = Gravatar(**gravatar_data)
|
||||
results.append(gravatar)
|
||||
self.email_gravatar_mapping.append((email, gravatar))
|
||||
|
||||
except Exception as e:
|
||||
Logger.error(
|
||||
@@ -85,13 +91,12 @@ class EmailToGravatarEnricher(Enricher):
|
||||
return results
|
||||
|
||||
def postprocess(self, results: List[OutputType], original_input: List[InputType]) -> List[OutputType]:
|
||||
for email_obj, gravatar_obj in zip(original_input, results):
|
||||
for email_obj, gravatar_obj in self.email_gravatar_mapping:
|
||||
if not self._graph_service:
|
||||
continue
|
||||
# Create email node
|
||||
self.create_node(email_obj)
|
||||
# Create gravatar node
|
||||
gravatar_key = f"{email_obj.email}_{self.sketch_id}"
|
||||
self.create_node(gravatar_obj)
|
||||
# Create relationship between email and gravatar
|
||||
self.create_relationship(email_obj, gravatar_obj, "HAS_GRAVATAR")
|
||||
|
||||
@@ -32,6 +32,7 @@ class IpToAsnEnricher(Enricher):
|
||||
vault=vault,
|
||||
params=params,
|
||||
)
|
||||
self.ip_asn_mapping: List[tuple[Ip, ASN]] = []
|
||||
|
||||
@classmethod
|
||||
def required_params(cls) -> bool:
|
||||
@@ -63,6 +64,7 @@ class IpToAsnEnricher(Enricher):
|
||||
|
||||
async def scan(self, data: List[InputType]) -> List[OutputType]:
|
||||
results: List[OutputType] = []
|
||||
self.ip_asn_mapping = []
|
||||
asnmap = AsnmapTool()
|
||||
|
||||
# Retrieve API key from vault or environment
|
||||
@@ -84,6 +86,7 @@ class IpToAsnEnricher(Enricher):
|
||||
description=asn_data.get("as_name", ""),
|
||||
)
|
||||
results.append(asn)
|
||||
self.ip_asn_mapping.append((ip, asn))
|
||||
Logger.info(
|
||||
self.sketch_id,
|
||||
{
|
||||
@@ -110,8 +113,8 @@ class IpToAsnEnricher(Enricher):
|
||||
self, results: List[OutputType], input_data: List[InputType] = None
|
||||
) -> List[OutputType]:
|
||||
# Create Neo4j relationships between IPs and their corresponding ASNs
|
||||
if input_data and self._graph_service:
|
||||
for ip, asn in zip(input_data, results):
|
||||
if self._graph_service:
|
||||
for ip, asn in self.ip_asn_mapping:
|
||||
# Create IP node
|
||||
self.create_node(ip)
|
||||
# Create ASN node
|
||||
|
||||
@@ -33,6 +33,7 @@ class IpToFraudScore(Enricher):
|
||||
vault=vault,
|
||||
params=params,
|
||||
)
|
||||
self.ip_risk_mapping: List[tuple[Ip, RiskProfile]] = []
|
||||
|
||||
# @classmethod
|
||||
# def required_params(cls) -> bool:
|
||||
@@ -70,6 +71,7 @@ class IpToFraudScore(Enricher):
|
||||
|
||||
async def scan(self, data: List[InputType]) -> List[OutputType]:
|
||||
results: List[OutputType] = []
|
||||
self.ip_risk_mapping = []
|
||||
|
||||
api_username = self.get_secret("SCAMLYTICS_USERNAME", os.getenv("SCAMLYTICS_USERNAME"))
|
||||
api_key = self.get_secret("SCAMLYTICS_API_KEY", os.getenv("SCAMLYTICS_API_KEY"))
|
||||
@@ -98,7 +100,7 @@ class IpToFraudScore(Enricher):
|
||||
Logger.error(
|
||||
self.sketch_id,
|
||||
{
|
||||
"message": f"(IpToFraudScore) Request to Scamlytics failed (status): '{response_json["status"]}'."
|
||||
"message": f"(IpToFraudScore) Request to Scamlytics failed (status): '{response_json['status']}'."
|
||||
},
|
||||
)
|
||||
continue
|
||||
@@ -121,11 +123,11 @@ class IpToFraudScore(Enricher):
|
||||
if proxy_info.get(key):
|
||||
proxy_flags.append(label)
|
||||
|
||||
results.append(
|
||||
RiskProfile(
|
||||
entity_id=ip.address, entity_type="IP address", overall_risk_score=fraud_score, risk_level=fraud_risk, last_updated=datetime.datetime.utcnow().isoformat(), risk_factors=proxy_flags, source="Scamalytics"
|
||||
)
|
||||
)
|
||||
risk_profile = RiskProfile(
|
||||
entity_id=ip.address, entity_type="IP address", overall_risk_score=fraud_score, risk_level=fraud_risk, last_updated=datetime.datetime.utcnow().isoformat(), risk_factors=proxy_flags, source="Scamalytics"
|
||||
)
|
||||
results.append(risk_profile)
|
||||
self.ip_risk_mapping.append((ip, risk_profile))
|
||||
|
||||
except Exception as e:
|
||||
Logger.error(self.sketch_id, {"message": f"(IpToFraudScore) Exception while querying {ip.address}: {e}"})
|
||||
@@ -138,16 +140,15 @@ class IpToFraudScore(Enricher):
|
||||
if not self._graph_service:
|
||||
return results
|
||||
|
||||
if input_data and self._graph_service:
|
||||
for ip, risk_profile in zip(input_data, results):
|
||||
self.create_node(ip)
|
||||
self.create_node(risk_profile)
|
||||
for ip, risk_profile in self.ip_risk_mapping:
|
||||
self.create_node(ip)
|
||||
self.create_node(risk_profile)
|
||||
|
||||
# Create relationship
|
||||
self.create_relationship(ip, risk_profile, "HAS_RISK_PROFILE")
|
||||
self.log_graph_message(
|
||||
f"(IpToFraudScore) IP {ip.address} has risk level of {risk_profile.risk_level}"
|
||||
)
|
||||
# Create relationship
|
||||
self.create_relationship(ip, risk_profile, "HAS_RISK_PROFILE")
|
||||
self.log_graph_message(
|
||||
f"(IpToFraudScore) IP {ip.address} has risk level of {risk_profile.risk_level}"
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
@@ -34,9 +34,8 @@ class IgnorantEnricher(Enricher):
|
||||
results: List[OutputType] = []
|
||||
for phone_obj in data:
|
||||
try:
|
||||
cleaned_phone = is_valid_number(phone_obj.number)
|
||||
if cleaned_phone:
|
||||
result = await self._perform_ignorant_research(cleaned_phone)
|
||||
if is_valid_number(phone_obj.number):
|
||||
result = await self._perform_ignorant_research(phone_obj.number)
|
||||
results.append(result)
|
||||
else:
|
||||
results.append({"number": phone_obj.number, "error": "Invalid phone number"})
|
||||
|
||||
@@ -155,7 +155,7 @@ def get_root_domain(domain: str) -> str:
|
||||
return domain
|
||||
|
||||
|
||||
def is_valid_number(phone: str, region: str = "FR") -> None:
|
||||
def is_valid_number(phone: str, region: str = "FR") -> bool:
|
||||
"""
|
||||
Validates a phone number. Raises InvalidPhoneNumberError if invalid.
|
||||
- `region` should be ISO 3166-1 alpha-2 country code (e.g., 'FR' for France)
|
||||
@@ -167,6 +167,8 @@ def is_valid_number(phone: str, region: str = "FR") -> None:
|
||||
except NumberParseException:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def parse_asn(asn: str) -> int:
|
||||
if not is_valid_asn(asn):
|
||||
|
||||
Reference in New Issue
Block a user