9 Commits

Author SHA1 Message Date
dextmorgn
9fcc60ca63 fix(deps): upgrade Python dependencies to patch 27 known vulnerabilities
pip-audit reported 27 vulnerabilities across 12 packages, including
cryptography PYSEC-2026-36 (CVSS 9.8) used by the vault, a starlette
Host-header path bypass, and python-multipart arbitrary file write.

- cryptography 45.0.7 -> 46.0.7 (pin was capped at <46)
- fastapi 0.115 -> 0.136, pulling starlette 0.46 -> 1.2
- python-multipart 0.0.20 -> 0.0.32
- pyjwt, urllib3, aiohttp, lxml, idna, mako and transitive bumps
- dev tools: pytest 8 -> 9 (requires pytest-httpx >=0.36), black 25 -> 26

All four module test suites pass (458 tests). pip-audit now reports
no known vulnerabilities.
2026-06-05 08:57:19 +02:00
dextmorgn
259019f23c fix(build): force LF on container files via .gitattributes
Windows checkouts with core.autocrlf=true convert entrypoint.sh to
CRLF, which Docker copies into the image. The shebang then reads
#!/bin/sh\r and exec fails with 'no such file or directory'.
Also covers .env.example (trailing \r in env values) and Makefile.
2026-06-04 18:23:53 +02:00
dextmorgn
7c082d733e docs(readme): make Windows install commands cmd-compatible
PowerShell foreach/Test-Path loop fails when pasted into cmd.exe.
Plain copy commands work in both shells.
2026-06-04 18:05:19 +02:00
dextmorgn
de93ae3059 docs(readme): add Windows install path without Make
Windows users hit failures with make-based install. Replicate
check-env + build + up targets as plain PowerShell/docker compose
commands for prod and dev.
2026-06-04 18:00:35 +02:00
dextmorgn
c6eaf08b66 Merge pull request #163 from Mubashirrrr/fix/json-import-apostrophe-corruption
fix(import): preserve apostrophes in valid JSON imports
2026-06-03 20:34:42 +02:00
dextmorgn
f380674955 Merge pull request #165 from reconurge/fix/sse-auth-header
fix(sse): auth header
2026-06-03 13:42:17 +02:00
Mubashir Rahim
1fdbdbd094 fix(import): preserve apostrophes in valid JSON imports
parse_json unconditionally replaced every single quote with a double
quote (`.replace("'", '"')`) before calling json.loads. This corrupted
any valid JSON string value containing an apostrophe, e.g. a person's
surname "Sarah O'Brien" became "Sarah O"Brien", causing the parse to
fail with "Invalid JSON" and the whole import to be rejected.

Parse strict JSON first so well-formed payloads (including apostrophes
in string values) import correctly, and only fall back to the lenient
single-quote replacement for Python-dict-style payloads that are not
valid JSON on their own.

Adds regression tests covering an apostrophe-bearing JSON value and the
retained single-quoted fallback.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-03 14:34:22 +05:00
dextmorgn
0b60c81361 Merge pull request #155 from Ghraven/fix-investigation-aware-timestamp
fix: use aware timestamp for investigation updates
2026-06-03 09:30:35 +02:00
Ghraven
60cfa16b8d fix: use aware timestamp for investigation updates 2026-06-03 08:37:45 +08:00
12 changed files with 1385 additions and 1363 deletions

5
.gitattributes vendored Normal file
View File

@@ -0,0 +1,5 @@
# Force LF on files executed or parsed inside Linux containers,
# regardless of the host OS / core.autocrlf setting.
*.sh text eol=lf
.env.example text eol=lf
Makefile text eol=lf

View File

@@ -31,6 +31,8 @@ Flowsint is still in early development and definetly needs the help of the commu
Don't want to read ? Got it. Here's your install instructions:
### Linux / macOS
#### 1. Install pre-requisites
- Docker
@@ -44,6 +46,35 @@ cd flowsint
make prod
```
### Windows
No Make needed. Works in both **Command Prompt (cmd)** and **PowerShell**.
#### 1. Install pre-requisites
- [Docker Desktop](https://docs.docker.com/desktop/setup/install/windows-install/) (make sure it is **running** before the next step)
- Git
#### 2. Clone and set up environment files
```bat
git clone https://github.com/reconurge/flowsint.git
cd flowsint
copy .env.example .env
copy .env.example flowsint-api\.env
copy .env.example flowsint-core\.env
copy .env.example flowsint-app\.env
```
#### 3. Build and start
```bat
docker compose -f docker-compose.prod.yml up -d --build
```
### First login
Then go to [http://localhost:5173/register](http://localhost:5173/register) and create an account. There are no credentials or account by default.
@@ -144,12 +175,19 @@ flowsint-types (types)
### Run
Make sure you have **Make** installed.
**Linux / macOS** (requires **Make**):
```bash
make dev
```
**Windows** (cmd or PowerShell, no Make — create the `.env` files first, see [Get started](#windows)):
```bat
docker compose -f docker-compose.dev.yml up -d --build
docker compose -f docker-compose.dev.yml logs -f
```
### Development
The app is accessible at [http://localhost:5173](http://localhost:5173).

View File

@@ -9,7 +9,7 @@ dependencies = [
"flowsint-core",
"flowsint-types",
"flowsint-enrichers",
"fastapi[standard]>=0.115.0,<0.116.0",
"fastapi[standard]>=0.115.0,<1.0.0",
"uvicorn>=0.32.0,<0.33.0",
"redis>=5.0,<6.0",
"celery>=5.3,<6.0",
@@ -28,14 +28,14 @@ dependencies = [
"networkx>=2.6.3,<3.0.0",
"email-validator>=2.2.0,<3.0.0",
"mistralai>=1.9.3,<2.0.0",
"python-multipart>=0.0.20,<0.0.21",
"python-multipart>=0.0.27,<0.1.0",
"openpyxl>=3.1.2,<4.0.0",
"jsonschema>=4.25.1,<5.0.0",
]
[dependency-groups]
dev = [
"black>=25.0,<26.0",
"black>=26.3.1,<27.0",
"isort>=6.0,<7.0",
"flake8>=7.0,<8.0",
"mypy>=1.17,<2.0",

View File

@@ -23,18 +23,18 @@ dependencies = [
"sse-starlette>=1.8,<2.0",
"alembic==1.13.0",
"phonenumbers>=9.0.8,<10.0.0",
"python-multipart>=0.0.20,<0.0.21",
"python-multipart>=0.0.27,<0.1.0",
"docker>=7.1.0,<8.0.0",
"pytest>=8.4.2,<9.0.0",
"cryptography>=45.0.7,<46.0.0",
"pytest>=9.0.3,<10.0.0",
"cryptography>=46.0.7,<47.0.0",
"openpyxl>=3.1,<4.0",
]
[dependency-groups]
dev = [
"pytest-asyncio>=0.21,<0.22",
"pytest-httpx>=0.35,<0.36",
"black>=25.0,<26.0",
"pytest-asyncio>=0.21,<2.0",
"pytest-httpx>=0.36,<1.0",
"black>=26.3.1,<27.0",
"isort>=6.0,<7.0",
"flake8>=7.0,<8.0",
"mypy>=1.17,<2.0",

View File

@@ -4,7 +4,7 @@ Investigation service for managing investigations and user roles.
from typing import List, Optional
from uuid import UUID, uuid4
from datetime import datetime
from datetime import datetime, timezone
from sqlalchemy.orm import Session
@@ -109,7 +109,7 @@ class InvestigationService(BaseService):
investigation.name = name
investigation.description = description
investigation.status = status
investigation.last_updated_at = datetime.utcnow()
investigation.last_updated_at = datetime.now(timezone.utc)
self._commit()
self._refresh(investigation)

View File

@@ -23,8 +23,15 @@ def parse_json(
try:
file_bytes = file_bytes.lstrip()
entities: Dict[str, Entity] = {}
my_bytes_value = file_bytes.decode().replace("'", '"')
graph = json.loads(my_bytes_value)
decoded = file_bytes.decode()
# Parse strict JSON first so apostrophes inside string values (e.g.
# "Sarah O'Brien") are preserved. Only fall back to the lenient
# single-quote replacement for Python-dict-style payloads that are not
# valid JSON on their own.
try:
graph = json.loads(decoded)
except json.JSONDecodeError:
graph = json.loads(decoded.replace("'", '"'))
node_key = next((k for k in VALID_NODES_KEYS if k in graph), None)
edge_key = next((k for k in VALID_EDGES_KEYS if k in graph), None)
if node_key is None:

View File

@@ -65,3 +65,40 @@ def test_standard_json_import_without_label():
"""Test basic standard JSON import."""
results = parse_json(standard_json_without_label, max_preview_rows=100)
assert "Username" in results.entities
# Valid JSON whose string values contain apostrophes (very common in OSINT
# data, e.g. surnames like "O'Brien"). This is well-formed JSON and must import.
json_with_apostrophe = b"""{
"nodes": [
{"id": "1", "label": "Sarah O'Brien", "type": "individual"},
{"id": "2", "label": "Bob", "type": "individual"}
],
"edges": [
{"source": "1", "target": "2", "label": "KNOWS"}
]
}"""
def test_json_import_preserves_apostrophes_in_string_values():
"""Valid JSON with an apostrophe in a value must parse and keep the value.
Regression: the parser used to blindly replace every single quote with a
double quote, which corrupted valid JSON (turning "Sarah O'Brien" into
"Sarah O"Brien") and raised "Invalid JSON".
"""
results = parse_json(json_with_apostrophe, max_preview_rows=100)
assert "Individual" in results.entities
labels = {
str(preview.obj.nodeLabel)
for preview in results.entities["Individual"].results
}
assert "Sarah O'Brien" in labels
def test_json_import_still_accepts_single_quoted_payload():
"""Python-dict-style payloads (single-quoted) remain supported as fallback."""
single_quoted = b"{'nodes': [{'id': '1', 'label': 'Alice', 'type': 'individual'}], 'edges': []}"
results = parse_json(single_quoted, max_preview_rows=100)
assert "Individual" in results.entities

View File

@@ -7,6 +7,7 @@ from uuid import uuid4
from flowsint_core.core.services.analysis_service import AnalysisService
from flowsint_core.core.services.chat_service import ChatService
from flowsint_core.core.services.flow_service import FlowService
from flowsint_core.core.services.investigation_service import InvestigationService
def _assert_utc(value):
@@ -70,3 +71,27 @@ def test_flow_service_create_uses_timezone_aware_timestamps():
_assert_utc(flow.created_at)
_assert_utc(flow.last_updated_at)
def test_investigation_service_update_uses_timezone_aware_timestamp():
investigation = MagicMock()
investigation_repo = MagicMock()
investigation_repo.get_by_id.return_value = investigation
service = InvestigationService(
db=MagicMock(),
investigation_repo=investigation_repo,
sketch_repo=MagicMock(),
analysis_repo=MagicMock(),
profile_repo=MagicMock(),
)
service._check_permission = MagicMock()
service.update(
investigation_id=uuid4(),
user_id=uuid4(),
name="Updated investigation",
description="Updated description",
status="active",
)
_assert_utc(investigation.last_updated_at)

View File

@@ -28,9 +28,9 @@ dependencies = [
[dependency-groups]
dev = [
"pytest>=8.4.2,<9.0.0",
"pytest-asyncio>=0.21,<0.22",
"black>=25.0,<26.0",
"pytest>=9.0.3,<10.0.0",
"pytest-asyncio>=0.21,<2.0",
"black>=26.3.1,<27.0",
"isort>=6.0,<7.0",
"flake8>=7.0,<8.0",
"mypy>=1.17,<2.0",

View File

@@ -13,8 +13,8 @@ dependencies = [
[dependency-groups]
dev = [
"pytest>=8.4.2,<9.0.0",
"black>=25.0,<26.0",
"pytest>=9.0.3,<10.0.0",
"black>=26.3.1,<27.0",
"isort>=6.0,<7.0",
"flake8>=7.0,<8.0",
"mypy>=1.17,<2.0",

View File

@@ -10,7 +10,7 @@ dependencies = [
"flowsint-enrichers",
"flowsint-api",
"pydantic[email]>=2.11.7,<3.0.0",
"python-multipart>=0.0.20,<0.0.21",
"python-multipart>=0.0.27,<0.1.0",
"docker>=7.1.0,<8.0.0",
]

2596
uv.lock generated

File diff suppressed because it is too large Load Diff