refactor: remove occurences of scanner

flowsint-transforms/tests/transforms/ip/asn_to_cidrs.py

@@ -0,0 +1,77 @@
+from flowsint_transforms.ips.asn_to_cidrs import AsnToCidrsTransform
+from flowsint_types.asn import ASN
+
+transform = AsnToCidrsTransform("sketch_123", "scan_123")
+
+
+def test_preprocess_valid_asns():
+    asns = [
+        ASN(number=15169),
+        ASN(number=13335),
+    ]
+    result = transform.preprocess(asns)
+
+    result_numbers = [asn.number for asn in result]
+    expected_numbers = [asn.number for asn in asns]
+
+    assert result_numbers == expected_numbers
+
+
+def test_unprocessed_valid_asns():
+    asns = [
+        "15169",
+        "13335",
+    ]
+    result = transform.preprocess(asns)
+    result_asns = [asn for asn in result]
+    expected_asns = [ASN(number=int(asn)) for asn in asns]
+    assert result_asns == expected_asns
+
+
+def test_preprocess_invalid_asns():
+    asns = [
+        ASN(number=15169),
+        ASN(number=999999999999),  # Invalid ASN number
+        ASN(number=13335),
+    ]
+    result = transform.preprocess(asns)
+
+    result_numbers = [asn.number for asn in result]
+    assert 15169 in result_numbers
+    assert 13335 in result_numbers
+    assert 999999999999 not in result_numbers
+
+
+def test_preprocess_multiple_formats():
+    asns = [
+        {"number": 15169},
+        {"invalid_key": 13335},
+        ASN(number=13335),
+        "15169",
+    ]
+    result = transform.preprocess(asns)
+
+    result_numbers = [asn.number for asn in result]
+    assert 15169 in result_numbers
+    assert 13335 in result_numbers
+    assert (
+        "invalid_key" not in result_numbers
+    )  # Should be filtered out due to invalid key
+
+
+def test_schemas():
+    input_schema = transform.input_schema()
+    output_schema = transform.output_schema()
+
+    # Input schema should have number field
+    assert "properties" in input_schema
+    number_prop = next(
+        (prop for prop in input_schema["properties"] if prop["name"] == "number"), None
+    )
+    assert number_prop is not None
+    assert number_prop["type"] == "integer"
+
+    # Output schema should have network field
+    assert "properties" in output_schema
+    prop_names = [prop["name"] for prop in output_schema["properties"]]
+    assert "network" in prop_names

flowsint-transforms/tests/transforms/ip/cidr_to_ips.py

@@ -0,0 +1,122 @@
+from flowsint_transforms.ips.cidr_to_ips import CidrToIpsTransform
+from flowsint_types.cidr import CIDR
+from flowsint_types.ip import Ip
+from tests.logger import TestLogger
+
+logger = TestLogger()
+transform = CidrToIpsTransform("sketch_123", "scan_123", logger)
+
+
+def test_preprocess_valid_cidrs():
+    cidrs = [
+        CIDR(network="8.8.8.0/24"),
+        CIDR(network="1.1.1.0/24"),
+    ]
+    result = transform.preprocess(cidrs)
+
+    result_networks = [cidr.network for cidr in result]
+    expected_networks = [cidr.network for cidr in cidrs]
+
+    assert result_networks == expected_networks
+
+
+def test_preprocess_unprocessed_valid_cidrs():
+    cidrs = [
+        "8.8.8.0/24",
+        "1.1.1.0/24",
+    ]
+    result = transform.preprocess(cidrs)
+    result_cidrs = [c for c in result]
+    expected_cidrs = [CIDR(network=c) for c in cidrs]
+    assert result_cidrs == expected_cidrs
+
+
+def test_preprocess_invalid_cidrs():
+    cidrs = [
+        CIDR(network="8.8.8.0/24"),
+        "invalid-cidr",
+        "not-a-cidr",
+    ]
+    result = transform.preprocess(cidrs)
+    result_networks = [str(cidr.network) for cidr in result]
+    assert "8.8.8.0/24" in result_networks
+    assert "invalid-cidr" not in result_networks
+    assert "not-a-cidr" not in result_networks
+
+
+def test_preprocess_multiple_formats():
+    cidrs = [
+        {"network": "8.8.8.0/24"},
+        {"invalid_key": "1.1.1.0/24"},
+        CIDR(network="9.9.9.0/24"),
+        "InvalidCIDR",
+    ]
+    result = transform.preprocess(cidrs)
+    result_networks = [str(cidr.network) for cidr in result]
+    assert "8.8.8.0/24" in result_networks
+    assert "9.9.9.0/24" in result_networks
+    assert "1.1.1.0/24" not in result_networks
+    assert "InvalidCIDR" not in result_networks
+
+
+def test_scan_extracts_ips(monkeypatch):
+    mock_dnsx_output = """8.35.200.12
+8.35.200.112
+8.35.200.16
+8.35.200.170"""
+
+    class MockSubprocessResult:
+        def __init__(self, stdout):
+            self.stdout = stdout
+            self.returncode = 0
+
+    def mock_subprocess_run(cmd, shell, capture_output, text, timeout):
+        assert "dnsx" in cmd
+        assert "-ptr" in cmd
+        return MockSubprocessResult(mock_dnsx_output)
+
+    # Patch the subprocess call in the transform
+    monkeypatch.setattr("subprocess.run", mock_subprocess_run)
+
+    input_data = [CIDR(network="8.35.200.0/24")]
+    ips = transform.scan(input_data)
+
+    assert isinstance(ips, list)
+    assert len(ips) == 4
+
+    expected_ips = ["8.35.200.12", "8.35.200.112", "8.35.200.16", "8.35.200.170"]
+
+    for ip in ips:
+        assert isinstance(ip, Ip)
+        assert ip.address in expected_ips
+
+
+def test_scan_handles_empty_output(monkeypatch):
+    class MockSubprocessResult:
+        def __init__(self):
+            self.stdout = ""
+            self.returncode = 0
+
+    def mock_subprocess_run(cmd, shell, capture_output, text, timeout):
+        return MockSubprocessResult()
+
+    monkeypatch.setattr("subprocess.run", mock_subprocess_run)
+
+    input_data = [CIDR(network="8.8.8.0/24")]
+    ips = transform.scan(input_data)
+
+    assert isinstance(ips, list)
+    assert len(ips) == 0
+
+
+def test_scan_handles_subprocess_exception(monkeypatch):
+    def mock_subprocess_run(cmd, shell, capture_output, text, timeout):
+        raise Exception("Subprocess failed")
+
+    monkeypatch.setattr("subprocess.run", mock_subprocess_run)
+
+    input_data = [CIDR(network="8.8.8.0/24")]
+    ips = transform.scan(input_data)
+
+    assert isinstance(ips, list)
+    assert len(ips) == 0

flowsint-transforms/tests/transforms/ip/ip_to_asn.py

@@ -0,0 +1,267 @@
+import json
+from unittest.mock import Mock
+from flowsint_transforms.ips.ip_to_asn import IpToAsnTransform
+from flowsint_types.ip import Ip
+from flowsint_types.asn import ASN
+from flowsint_types.cidr import CIDR
+from tests.logger import TestLogger
+
+logger = TestLogger()
+# The transform will get a mock logger from conftest.py automatically
+transform = IpToAsnTransform("sketch_123", "scan_123", logger)
+
+
+def test_preprocess_valid_ips():
+    ips = [
+        Ip(address="8.8.8.8"),
+        Ip(address="1.1.1.1"),
+    ]
+    result = transform.preprocess(ips)
+
+    result_addresses = [ip.address for ip in result]
+    expected_addresses = [ip.address for ip in ips]
+
+    assert result_addresses == expected_addresses
+
+
+def test_unprocessed_valid_ips():
+    ips = [
+        "8.8.8.8",
+        "1.1.1.1",
+    ]
+    result = transform.preprocess(ips)
+    result_ips = [ip for ip in result]
+    expected_ips = [Ip(address=ip) for ip in ips]
+    assert result_ips == expected_ips
+
+
+def test_preprocess_invalid_ips():
+    ips = [
+        Ip(address="8.8.8.8"),
+        Ip(address="invalid_ip"),
+        Ip(address="192.168.1.1"),
+    ]
+    result = transform.preprocess(ips)
+
+    result_addresses = [ip.address for ip in result]
+    assert "8.8.8.8" in result_addresses
+    assert "192.168.1.1" in result_addresses
+    assert "invalid_ip" not in result_addresses
+
+
+def test_preprocess_multiple_formats():
+    ips = [
+        {"address": "8.8.8.8"},
+        {"invalid_key": "1.1.1.1"},
+        Ip(address="192.168.1.1"),
+        "10.0.0.1",
+    ]
+    result = transform.preprocess(ips)
+
+    result_addresses = [ip.address for ip in result]
+    assert "8.8.8.8" in result_addresses
+    assert "192.168.1.1" in result_addresses
+    assert "10.0.0.1" in result_addresses
+    assert (
+        "1.1.1.1" not in result_addresses
+    )  # Should be filtered out due to invalid key
+
+
+def test_scan_extracts_asn_info(monkeypatch):
+    mock_asnmap_output = {
+        "input": "8.8.8.8",
+        "as_number": "AS15169",
+        "as_name": "GOOGLE",
+        "as_country": "US",
+        "as_range": ["8.8.8.0/24", "8.8.4.0/24"],
+    }
+
+    class MockSubprocessResult:
+        def __init__(self, stdout):
+            self.stdout = stdout
+            self.returncode = 0
+
+    def mock_subprocess_run(cmd, input, capture_output, text, timeout):
+        assert "asnmap" in cmd
+        assert input == "8.8.8.8"
+        return MockSubprocessResult(json.dumps(mock_asnmap_output))
+
+    # Patch the subprocess call in the transform
+    monkeypatch.setattr("subprocess.run", mock_subprocess_run)
+
+    input_data = [Ip(address="8.8.8.8")]
+    asns = transform.scan(input_data)
+
+    assert isinstance(asns, list)
+    assert len(asns) == 1
+
+    asn = asns[0]
+    assert isinstance(asn, ASN)
+    assert asn.number == 15169  # AS15169 -> 15169
+    assert asn.name == "GOOGLE"
+    assert asn.country == "US"
+    assert len(asn.cidrs) == 2
+    assert str(asn.cidrs[0].network) == "8.8.8.0/24"
+    assert str(asn.cidrs[1].network) == "8.8.4.0/24"
+
+
+def test_scan_handles_no_asn_found(monkeypatch):
+    class MockSubprocessResult:
+        def __init__(self, stdout):
+            self.stdout = stdout
+            self.returncode = 0
+
+    def mock_subprocess_run(cmd, input, capture_output, text, timeout):
+        # Return empty output to simulate no ASN found
+        return MockSubprocessResult("")
+
+    monkeypatch.setattr("subprocess.run", mock_subprocess_run)
+
+    input_data = [Ip(address="192.168.1.1")]
+    asns = transform.scan(input_data)
+
+    assert isinstance(asns, list)
+    assert len(asns) == 1
+
+    asn = asns[0]
+    assert isinstance(asn, ASN)
+    assert asn.number == 0
+    assert asn.name == "Unknown"
+    assert asn.country == "Unknown"
+    assert len(asn.cidrs) == 0
+
+
+def test_scan_handles_subprocess_exception(monkeypatch):
+    def mock_subprocess_run(cmd, input, capture_output, text, timeout):
+        raise Exception("Subprocess failed")
+
+    monkeypatch.setattr("subprocess.run", mock_subprocess_run)
+
+    input_data = [Ip(address="8.8.8.8")]
+    asns = transform.scan(input_data)
+
+    assert isinstance(asns, list)
+    assert len(asns) == 1
+
+    asn = asns[0]
+    assert isinstance(asn, ASN)
+    assert asn.number == 0
+    assert asn.name == "Unknown"
+    assert asn.country == "Unknown"
+
+
+def test_scan_multiple_ips(monkeypatch):
+    mock_responses = {
+        "8.8.8.8": {
+            "input": "8.8.8.8",
+            "as_number": "AS15169",
+            "as_name": "GOOGLE",
+            "as_country": "US",
+            "as_range": ["8.8.8.0/24"],
+        },
+        "1.1.1.1": {
+            "input": "1.1.1.1",
+            "as_number": "AS13335",
+            "as_name": "CLOUDFLARE",
+            "as_country": "US",
+            "as_range": ["1.1.1.0/24"],
+        },
+    }
+
+    class MockSubprocessResult:
+        def __init__(self, stdout):
+            self.stdout = stdout
+            self.returncode = 0
+
+    def mock_subprocess_run(cmd, input, capture_output, text, timeout):
+        if input in mock_responses:
+            return MockSubprocessResult(json.dumps(mock_responses[input]))
+        return MockSubprocessResult("")
+
+    monkeypatch.setattr("subprocess.run", mock_subprocess_run)
+
+    input_data = [Ip(address="8.8.8.8"), Ip(address="1.1.1.1")]
+    asns = transform.scan(input_data)
+
+    assert len(asns) == 2
+
+    # Check first ASN
+    assert asns[0].number == 15169
+    assert asns[0].name == "GOOGLE"
+
+    # Check second ASN
+    assert asns[1].number == 13335
+    assert asns[1].name == "CLOUDFLARE"
+
+
+def test_schemas():
+    input_schema = transform.input_schema()
+    output_schema = transform.output_schema()
+
+    # Input schema should have address field
+    assert "properties" in input_schema
+    address_prop = next(
+        (prop for prop in input_schema["properties"] if prop["name"] == "address"), None
+    )
+    assert address_prop is not None
+    assert address_prop["type"] == "string"
+
+    # Output schema should have ASN fields
+    assert "properties" in output_schema
+    prop_names = [prop["name"] for prop in output_schema["properties"]]
+    assert "number" in prop_names
+    assert "name" in prop_names
+    assert "country" in prop_names
+    assert "cidrs" in prop_names
+
+
+def test_postprocess_creates_neo4j_relationships(monkeypatch):
+    # Mock Neo4j connection
+    mock_neo4j = Mock()
+    transform.neo4j_conn = mock_neo4j
+
+    input_data = [Ip(address="8.8.8.8")]
+    asn_results = [
+        ASN(
+            number=15169,
+            name="GOOGLE",
+            country="US",
+            cidrs=[CIDR(network="8.8.8.0/24")],
+        )
+    ]
+
+    result = transform.postprocess(asn_results, input_data)
+
+    # Verify Neo4j query was called
+    mock_neo4j.query.assert_called_once()
+
+    # Check the query parameters
+    call_args = mock_neo4j.query.call_args
+    params = call_args[0][1]
+    assert params["ip_address"] == "8.8.8.8"
+    assert params["asn_number"] == 15169
+    assert params["asn_name"] == "GOOGLE"
+    assert params["asn_country"] == "US"
+    assert params["sketch_id"] == "sketch_123"
+
+    # Should return the same results
+    assert result == asn_results
+
+
+def test_postprocess_skips_unknown_asns(monkeypatch):
+    # Mock Neo4j connection
+    mock_neo4j = Mock()
+    transform.neo4j_conn = mock_neo4j
+
+    input_data = [Ip(address="192.168.1.1")]
+    asn_results = [
+        ASN(number=0, name="Unknown", country="Unknown", cidrs=[])  # Unknown ASN
+    ]
+
+    result = transform.postprocess(asn_results, input_data)
+
+    # Verify Neo4j query was NOT called for unknown ASN
+    mock_neo4j.query.assert_not_called()
+
+    # Should return the same results
+    assert result == asn_results

flowsint-transforms/tests/transforms/ip/ip_to_infos.py

@@ -0,0 +1,103 @@
+from flowsint_transforms.ips.geolocation import IpToInfosTransform
+from flowsint_types.ip import Ip, Ip
+
+transform = IpToInfosTransform("sketch_123", "scan_123")
+
+
+def test_preprocess_valid_ips():
+    ips = [
+        Ip(address="8.8.8.8"),
+        Ip(address="1.1.1.1"),
+    ]
+    result = transform.preprocess(ips)
+    result_ips = [d.address for d in result]
+    expected_ips = [d.address for d in ips]
+    assert result_ips == expected_ips
+
+
+def test_preprocess_string_ips():
+    ips = [
+        "8.8.8.8",
+        "1.1.1.1",
+    ]
+    result = transform.preprocess(ips)
+    result_ips = [d.address for d in result]
+    expected_ips = [d for d in ips]
+    assert [ip.address for ip in result] == expected_ips
+
+
+def test_preprocess_invalid_ips():
+    ips = [
+        Ip(address="8.8.8.8"),
+        Ip(address="invalid_ip"),
+        Ip(address="1.1.1.1"),
+    ]
+    result = transform.preprocess(ips)
+    result_ips = [d.address for d in result]
+    assert "8.8.8.8" in result_ips
+    assert "1.1.1.1" in result_ips
+    assert "invalid_ip" not in result_ips
+
+
+def test_preprocess_multiple_formats():
+    ips = [
+        {"address": "8.8.8.8"},
+        {"invalid_key": "1.2.3.4"},
+        Ip(address="1.1.1.1"),
+        "1.1.1.1",
+    ]
+    result = transform.preprocess(ips)
+    result_ips = [d.address for d in result]
+    assert "8.8.8.8" in result_ips
+    assert "1.1.1.1" in result_ips
+    assert "1.2.3.4" not in result_ips
+
+
+def test_scan_returns_ip(monkeypatch):
+    # Mock of get_location_data
+    def mock_get_location_data(address):
+        return {
+            "latitude": 37.386,
+            "longitude": -122.0838,
+            "country": "US",
+            "city": "Mountain View",
+            "isp": "Google LLC",
+        }
+
+    monkeypatch.setattr(transform, "get_location_data", mock_get_location_data)
+
+    input_data = [Ip(address="8.8.8.8")]
+    output = transform.execute(input_data)
+    assert isinstance(output, list)
+    assert isinstance(output[0], Ip)
+    assert output[0].address == "8.8.8.8"
+    assert output[0].city == "Mountain View"
+    assert output[0].country == "US"
+    assert output[0].isp == "Google LLC"
+
+
+def test_schemas():
+    input_schema = transform.input_schema()
+    output_schema = transform.output_schema()
+    assert input_schema == {
+        "type": "Ip",
+        "properties": [
+            {"name": "address", "type": "string"},
+            {"name": "latitude", "type": "number | null"},
+            {"name": "longitude", "type": "number | null"},
+            {"name": "country", "type": "string | null"},
+            {"name": "city", "type": "string | null"},
+            {"name": "isp", "type": "string | null"},
+        ],
+    }
+    assert output_schema == {
+        "type": "Ip",
+        "properties": [
+            {"name": "address", "type": "string"},
+            {"name": "latitude", "type": "number | null"},
+            {"name": "longitude", "type": "number | null"},
+            {"name": "country", "type": "string | null"},
+            {"name": "city", "type": "string | null"},
+            {"name": "isp", "type": "string | null"},
+        ],
+    }