mirror of
https://github.com/reconurge/flowsint.git
synced 2026-07-11 13:35:22 -05:00
Compare commits
9 Commits
fix/sse-au
...
chore/secu
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9fcc60ca63 | ||
|
|
259019f23c | ||
|
|
7c082d733e | ||
|
|
de93ae3059 | ||
|
|
c6eaf08b66 | ||
|
|
f380674955 | ||
|
|
1fdbdbd094 | ||
|
|
0b60c81361 | ||
|
|
60cfa16b8d |
5
.gitattributes
vendored
Normal file
5
.gitattributes
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
# Force LF on files executed or parsed inside Linux containers,
|
||||
# regardless of the host OS / core.autocrlf setting.
|
||||
*.sh text eol=lf
|
||||
.env.example text eol=lf
|
||||
Makefile text eol=lf
|
||||
40
README.md
40
README.md
@@ -31,6 +31,8 @@ Flowsint is still in early development and definetly needs the help of the commu
|
||||
|
||||
Don't want to read ? Got it. Here's your install instructions:
|
||||
|
||||
### Linux / macOS
|
||||
|
||||
#### 1. Install pre-requisites
|
||||
|
||||
- Docker
|
||||
@@ -44,6 +46,35 @@ cd flowsint
|
||||
make prod
|
||||
```
|
||||
|
||||
### Windows
|
||||
|
||||
No Make needed. Works in both **Command Prompt (cmd)** and **PowerShell**.
|
||||
|
||||
#### 1. Install pre-requisites
|
||||
|
||||
- [Docker Desktop](https://docs.docker.com/desktop/setup/install/windows-install/) (make sure it is **running** before the next step)
|
||||
- Git
|
||||
|
||||
#### 2. Clone and set up environment files
|
||||
|
||||
```bat
|
||||
git clone https://github.com/reconurge/flowsint.git
|
||||
cd flowsint
|
||||
|
||||
copy .env.example .env
|
||||
copy .env.example flowsint-api\.env
|
||||
copy .env.example flowsint-core\.env
|
||||
copy .env.example flowsint-app\.env
|
||||
```
|
||||
|
||||
#### 3. Build and start
|
||||
|
||||
```bat
|
||||
docker compose -f docker-compose.prod.yml up -d --build
|
||||
```
|
||||
|
||||
### First login
|
||||
|
||||
Then go to [http://localhost:5173/register](http://localhost:5173/register) and create an account. There are no credentials or account by default.
|
||||
|
||||
|
||||
@@ -144,12 +175,19 @@ flowsint-types (types)
|
||||
|
||||
### Run
|
||||
|
||||
Make sure you have **Make** installed.
|
||||
**Linux / macOS** (requires **Make**):
|
||||
|
||||
```bash
|
||||
make dev
|
||||
```
|
||||
|
||||
**Windows** (cmd or PowerShell, no Make — create the `.env` files first, see [Get started](#windows)):
|
||||
|
||||
```bat
|
||||
docker compose -f docker-compose.dev.yml up -d --build
|
||||
docker compose -f docker-compose.dev.yml logs -f
|
||||
```
|
||||
|
||||
### Development
|
||||
|
||||
The app is accessible at [http://localhost:5173](http://localhost:5173).
|
||||
|
||||
@@ -9,7 +9,7 @@ dependencies = [
|
||||
"flowsint-core",
|
||||
"flowsint-types",
|
||||
"flowsint-enrichers",
|
||||
"fastapi[standard]>=0.115.0,<0.116.0",
|
||||
"fastapi[standard]>=0.115.0,<1.0.0",
|
||||
"uvicorn>=0.32.0,<0.33.0",
|
||||
"redis>=5.0,<6.0",
|
||||
"celery>=5.3,<6.0",
|
||||
@@ -28,14 +28,14 @@ dependencies = [
|
||||
"networkx>=2.6.3,<3.0.0",
|
||||
"email-validator>=2.2.0,<3.0.0",
|
||||
"mistralai>=1.9.3,<2.0.0",
|
||||
"python-multipart>=0.0.20,<0.0.21",
|
||||
"python-multipart>=0.0.27,<0.1.0",
|
||||
"openpyxl>=3.1.2,<4.0.0",
|
||||
"jsonschema>=4.25.1,<5.0.0",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"black>=25.0,<26.0",
|
||||
"black>=26.3.1,<27.0",
|
||||
"isort>=6.0,<7.0",
|
||||
"flake8>=7.0,<8.0",
|
||||
"mypy>=1.17,<2.0",
|
||||
|
||||
@@ -23,18 +23,18 @@ dependencies = [
|
||||
"sse-starlette>=1.8,<2.0",
|
||||
"alembic==1.13.0",
|
||||
"phonenumbers>=9.0.8,<10.0.0",
|
||||
"python-multipart>=0.0.20,<0.0.21",
|
||||
"python-multipart>=0.0.27,<0.1.0",
|
||||
"docker>=7.1.0,<8.0.0",
|
||||
"pytest>=8.4.2,<9.0.0",
|
||||
"cryptography>=45.0.7,<46.0.0",
|
||||
"pytest>=9.0.3,<10.0.0",
|
||||
"cryptography>=46.0.7,<47.0.0",
|
||||
"openpyxl>=3.1,<4.0",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest-asyncio>=0.21,<0.22",
|
||||
"pytest-httpx>=0.35,<0.36",
|
||||
"black>=25.0,<26.0",
|
||||
"pytest-asyncio>=0.21,<2.0",
|
||||
"pytest-httpx>=0.36,<1.0",
|
||||
"black>=26.3.1,<27.0",
|
||||
"isort>=6.0,<7.0",
|
||||
"flake8>=7.0,<8.0",
|
||||
"mypy>=1.17,<2.0",
|
||||
|
||||
@@ -4,7 +4,7 @@ Investigation service for managing investigations and user roles.
|
||||
|
||||
from typing import List, Optional
|
||||
from uuid import UUID, uuid4
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -109,7 +109,7 @@ class InvestigationService(BaseService):
|
||||
investigation.name = name
|
||||
investigation.description = description
|
||||
investigation.status = status
|
||||
investigation.last_updated_at = datetime.utcnow()
|
||||
investigation.last_updated_at = datetime.now(timezone.utc)
|
||||
|
||||
self._commit()
|
||||
self._refresh(investigation)
|
||||
|
||||
@@ -23,8 +23,15 @@ def parse_json(
|
||||
try:
|
||||
file_bytes = file_bytes.lstrip()
|
||||
entities: Dict[str, Entity] = {}
|
||||
my_bytes_value = file_bytes.decode().replace("'", '"')
|
||||
graph = json.loads(my_bytes_value)
|
||||
decoded = file_bytes.decode()
|
||||
# Parse strict JSON first so apostrophes inside string values (e.g.
|
||||
# "Sarah O'Brien") are preserved. Only fall back to the lenient
|
||||
# single-quote replacement for Python-dict-style payloads that are not
|
||||
# valid JSON on their own.
|
||||
try:
|
||||
graph = json.loads(decoded)
|
||||
except json.JSONDecodeError:
|
||||
graph = json.loads(decoded.replace("'", '"'))
|
||||
node_key = next((k for k in VALID_NODES_KEYS if k in graph), None)
|
||||
edge_key = next((k for k in VALID_EDGES_KEYS if k in graph), None)
|
||||
if node_key is None:
|
||||
|
||||
@@ -65,3 +65,40 @@ def test_standard_json_import_without_label():
|
||||
"""Test basic standard JSON import."""
|
||||
results = parse_json(standard_json_without_label, max_preview_rows=100)
|
||||
assert "Username" in results.entities
|
||||
|
||||
|
||||
# Valid JSON whose string values contain apostrophes (very common in OSINT
|
||||
# data, e.g. surnames like "O'Brien"). This is well-formed JSON and must import.
|
||||
json_with_apostrophe = b"""{
|
||||
"nodes": [
|
||||
{"id": "1", "label": "Sarah O'Brien", "type": "individual"},
|
||||
{"id": "2", "label": "Bob", "type": "individual"}
|
||||
],
|
||||
"edges": [
|
||||
{"source": "1", "target": "2", "label": "KNOWS"}
|
||||
]
|
||||
}"""
|
||||
|
||||
|
||||
def test_json_import_preserves_apostrophes_in_string_values():
|
||||
"""Valid JSON with an apostrophe in a value must parse and keep the value.
|
||||
|
||||
Regression: the parser used to blindly replace every single quote with a
|
||||
double quote, which corrupted valid JSON (turning "Sarah O'Brien" into
|
||||
"Sarah O"Brien") and raised "Invalid JSON".
|
||||
"""
|
||||
results = parse_json(json_with_apostrophe, max_preview_rows=100)
|
||||
|
||||
assert "Individual" in results.entities
|
||||
labels = {
|
||||
str(preview.obj.nodeLabel)
|
||||
for preview in results.entities["Individual"].results
|
||||
}
|
||||
assert "Sarah O'Brien" in labels
|
||||
|
||||
|
||||
def test_json_import_still_accepts_single_quoted_payload():
|
||||
"""Python-dict-style payloads (single-quoted) remain supported as fallback."""
|
||||
single_quoted = b"{'nodes': [{'id': '1', 'label': 'Alice', 'type': 'individual'}], 'edges': []}"
|
||||
results = parse_json(single_quoted, max_preview_rows=100)
|
||||
assert "Individual" in results.entities
|
||||
|
||||
@@ -7,6 +7,7 @@ from uuid import uuid4
|
||||
from flowsint_core.core.services.analysis_service import AnalysisService
|
||||
from flowsint_core.core.services.chat_service import ChatService
|
||||
from flowsint_core.core.services.flow_service import FlowService
|
||||
from flowsint_core.core.services.investigation_service import InvestigationService
|
||||
|
||||
|
||||
def _assert_utc(value):
|
||||
@@ -70,3 +71,27 @@ def test_flow_service_create_uses_timezone_aware_timestamps():
|
||||
|
||||
_assert_utc(flow.created_at)
|
||||
_assert_utc(flow.last_updated_at)
|
||||
|
||||
|
||||
def test_investigation_service_update_uses_timezone_aware_timestamp():
|
||||
investigation = MagicMock()
|
||||
investigation_repo = MagicMock()
|
||||
investigation_repo.get_by_id.return_value = investigation
|
||||
service = InvestigationService(
|
||||
db=MagicMock(),
|
||||
investigation_repo=investigation_repo,
|
||||
sketch_repo=MagicMock(),
|
||||
analysis_repo=MagicMock(),
|
||||
profile_repo=MagicMock(),
|
||||
)
|
||||
service._check_permission = MagicMock()
|
||||
|
||||
service.update(
|
||||
investigation_id=uuid4(),
|
||||
user_id=uuid4(),
|
||||
name="Updated investigation",
|
||||
description="Updated description",
|
||||
status="active",
|
||||
)
|
||||
|
||||
_assert_utc(investigation.last_updated_at)
|
||||
|
||||
@@ -28,9 +28,9 @@ dependencies = [
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8.4.2,<9.0.0",
|
||||
"pytest-asyncio>=0.21,<0.22",
|
||||
"black>=25.0,<26.0",
|
||||
"pytest>=9.0.3,<10.0.0",
|
||||
"pytest-asyncio>=0.21,<2.0",
|
||||
"black>=26.3.1,<27.0",
|
||||
"isort>=6.0,<7.0",
|
||||
"flake8>=7.0,<8.0",
|
||||
"mypy>=1.17,<2.0",
|
||||
|
||||
@@ -13,8 +13,8 @@ dependencies = [
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8.4.2,<9.0.0",
|
||||
"black>=25.0,<26.0",
|
||||
"pytest>=9.0.3,<10.0.0",
|
||||
"black>=26.3.1,<27.0",
|
||||
"isort>=6.0,<7.0",
|
||||
"flake8>=7.0,<8.0",
|
||||
"mypy>=1.17,<2.0",
|
||||
|
||||
@@ -10,7 +10,7 @@ dependencies = [
|
||||
"flowsint-enrichers",
|
||||
"flowsint-api",
|
||||
"pydantic[email]>=2.11.7,<3.0.0",
|
||||
"python-multipart>=0.0.20,<0.0.21",
|
||||
"python-multipart>=0.0.27,<0.1.0",
|
||||
"docker>=7.1.0,<8.0.0",
|
||||
]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user