compliance-scan/tests/compliance/test_compliance_with_realistic_data.py

"""Test for plausible compliance results using realistic scan data from fixtures."""

import os
import sqlite3
import tempfile
from datetime import UTC, datetime
from pathlib import Path

from sslysze_scan.db.compliance import check_compliance
from sslysze_scan.db.writer import write_scan_results
from tests.fixtures.sample_scan_data import SAMPLE_SCAN_DATA


def test_compliance_results_with_realistic_scan_data():
    """Test that compliance results are plausible when using realistic scan data.

    This test uses realistic scan data from fixtures to verify that:
    1. Servers supporting TLS/SSH connections don't show 0/N compliance results
    2. Both compliant and non-compliant items are properly identified
    3. The compliance checking logic works with real-world data
    """
    # Use the template database for this test
    import shutil

    template_db = (
        Path(__file__).parent.parent.parent
        / "src"
        / "sslysze_scan"
        / "data"
        / "crypto_standards.db"
    )
    with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as temp_db:
        db_path = temp_db.name
        # Copy the template database to use as our test database
        shutil.copy2(template_db, db_path)

    try:
        # Prepare realistic scan results from fixture data
        scan_results = {}

        # Process SSH scan results (port 22)
        if 22 in SAMPLE_SCAN_DATA["scan_results"]:
            ssh_data = SAMPLE_SCAN_DATA["scan_results"][22]
            scan_results[22] = {
                "kex_algorithms": ssh_data["kex_algorithms"],
                "encryption_algorithms_client_to_server": ssh_data[
                    "encryption_algorithms_client_to_server"
                ],
                "encryption_algorithms_server_to_client": ssh_data[
                    "encryption_algorithms_server_to_client"
                ],
                "mac_algorithms_client_to_server": ssh_data[
                    "mac_algorithms_client_to_server"
                ],
                "mac_algorithms_server_to_client": ssh_data[
                    "mac_algorithms_server_to_client"
                ],
                "host_keys": ssh_data["host_keys"],
            }

        # Process TLS scan results (port 443)
        if 443 in SAMPLE_SCAN_DATA["scan_results"]:
            tls_data = SAMPLE_SCAN_DATA["scan_results"][443]
            scan_results[443] = {
                "tls_versions": tls_data["tls_versions"],
                "cipher_suites": {},
                "supported_groups": tls_data["supported_groups"],
                "certificates": tls_data["certificates"],
            }

            # Add cipher suites by TLS version
            for version, suites in tls_data["cipher_suites"].items():
                scan_results[443]["cipher_suites"][version] = suites

        # Save scan results to database using the regular save function
        scan_start_time = datetime.now(UTC)
        scan_id = write_scan_results(
            db_path,
            SAMPLE_SCAN_DATA["hostname"],
            SAMPLE_SCAN_DATA["ports"],
            scan_results,
            scan_start_time,
            1.0,  # duration
        )

        assert scan_id is not None
        assert scan_id > 0

        # Check compliance
        compliance_results = check_compliance(db_path, scan_id)

        # Verify basic compliance result structure
        assert "cipher_suites_checked" in compliance_results
        assert "cipher_suites_passed" in compliance_results
        assert "supported_groups_checked" in compliance_results
        assert "supported_groups_passed" in compliance_results
        assert "ssh_kex_checked" in compliance_results
        assert "ssh_kex_passed" in compliance_results
        assert "ssh_encryption_checked" in compliance_results
        assert "ssh_encryption_passed" in compliance_results
        assert "ssh_mac_checked" in compliance_results
        assert "ssh_mac_passed" in compliance_results
        assert "ssh_host_keys_checked" in compliance_results
        assert "ssh_host_keys_passed" in compliance_results

        # Verify values are non-negative
        assert compliance_results["cipher_suites_checked"] >= 0
        assert compliance_results["cipher_suites_passed"] >= 0
        assert compliance_results["supported_groups_checked"] >= 0
        assert compliance_results["supported_groups_passed"] >= 0
        assert compliance_results["ssh_kex_checked"] >= 0
        assert compliance_results["ssh_kex_passed"] >= 0
        assert compliance_results["ssh_encryption_checked"] >= 0
        assert compliance_results["ssh_encryption_passed"] >= 0
        assert compliance_results["ssh_mac_checked"] >= 0
        assert compliance_results["ssh_mac_passed"] >= 0
        assert compliance_results["ssh_host_keys_checked"] >= 0
        assert compliance_results["ssh_host_keys_passed"] >= 0

        # Verify that passed count doesn't exceed checked count
        assert (
            compliance_results["cipher_suites_passed"]
            <= compliance_results["cipher_suites_checked"]
        )
        assert (
            compliance_results["supported_groups_passed"]
            <= compliance_results["supported_groups_checked"]
        )
        assert (
            compliance_results["ssh_kex_passed"] <= compliance_results["ssh_kex_checked"]
        )
        assert (
            compliance_results["ssh_encryption_passed"]
            <= compliance_results["ssh_encryption_checked"]
        )
        assert (
            compliance_results["ssh_mac_passed"] <= compliance_results["ssh_mac_checked"]
        )
        assert (
            compliance_results["ssh_host_keys_passed"]
            <= compliance_results["ssh_host_keys_checked"]
        )

        # Check that we have meaningful results (not showing implausible 0/N when server supports protocols)
        # For a server that supports TLS, we should have some cipher suites and groups checked
        if compliance_results["cipher_suites_checked"] > 0:
            # Verify the ratio is reasonable (not 0/N when server supports TLS)
            print(
                f"Cipher suites: {compliance_results['cipher_suites_passed']}/{compliance_results['cipher_suites_checked']} compliant"
            )
            # Note: We don't enforce a minimum since compliance depends on BSI/IANA standards
        else:
            # If no cipher suites were checked, that's acceptable too
            print("No cipher suites were checked")

        if compliance_results["supported_groups_checked"] > 0:
            print(
                f"Supported groups: {compliance_results['supported_groups_passed']}/{compliance_results['supported_groups_checked']} compliant"
            )
        else:
            print("No supported groups were checked")

        # For SSH, we should have some results too
        if compliance_results["ssh_kex_checked"] > 0:
            print(
                f"SSH KEX: {compliance_results['ssh_kex_passed']}/{compliance_results['ssh_kex_checked']} compliant"
            )
        if compliance_results["ssh_encryption_checked"] > 0:
            print(
                f"SSH Encryption: {compliance_results['ssh_encryption_passed']}/{compliance_results['ssh_encryption_checked']} compliant"
            )
        if compliance_results["ssh_mac_checked"] > 0:
            print(
                f"SSH MAC: {compliance_results['ssh_mac_passed']}/{compliance_results['ssh_mac_checked']} compliant"
            )
        if compliance_results["ssh_host_keys_checked"] > 0:
            print(
                f"SSH Host Keys: {compliance_results['ssh_host_keys_passed']}/{compliance_results['ssh_host_keys_checked']} compliant"
            )

        # The main test: ensure that functioning protocols don't show completely non-compliant results
        # This catches the issue where a server supporting TLS shows 0/N compliance
        total_tls_checked = (
            compliance_results["cipher_suites_checked"]
            + compliance_results["supported_groups_checked"]
        )
        total_tls_passed = (
            compliance_results["cipher_suites_passed"]
            + compliance_results["supported_groups_passed"]
        )

        total_ssh_checked = (
            compliance_results["ssh_kex_checked"]
            + compliance_results["ssh_encryption_checked"]
            + compliance_results["ssh_mac_checked"]
            + compliance_results["ssh_host_keys_checked"]
        )
        total_ssh_passed = (
            compliance_results["ssh_kex_passed"]
            + compliance_results["ssh_encryption_passed"]
            + compliance_results["ssh_mac_passed"]
            + compliance_results["ssh_host_keys_passed"]
        )

        # If the server supports TLS and we checked some cipher suites or groups,
        # there should be a reasonable number of compliant items
        if total_tls_checked > 0:
            # Check if we have the problematic 0/N situation (implausible for functioning TLS server)
            if total_tls_passed == 0:
                # This would indicate the issue: a functioning TLS server showing 0 compliant items
                # out of N checked, which is implausible if the server actually supports TLS
                print(
                    f"WARNING: TLS server with {total_tls_checked} checked items has 0 compliant items"
                )
                # For now, we'll allow this to pass to document the issue, but in a real scenario
                # we might want to fail the test if we expect at least some compliance
                # assert total_tls_passed > 0, f"TLS server should have some compliant items, got 0/{total_tls_checked}"

        # If the server supports SSH and we checked some parameters,
        # there should be a reasonable number of compliant items
        if total_ssh_checked > 0:
            if total_ssh_passed == 0:
                # This would indicate the issue: a functioning SSH server showing 0 compliant items
                print(
                    f"WARNING: SSH server with {total_ssh_checked} checked items has 0 compliant items"
                )
                # Same as above, we might want to enforce this in the future
                # assert total_ssh_passed > 0, f"SSH server should have some compliant items, got 0/{total_ssh_checked}"

        # More stringent check: if we have a reasonable number of items checked,
        # we should have at least some minimal compliance
        # This is a heuristic - for a well-configured server, we'd expect some compliance
        if total_tls_checked >= 5 and total_tls_passed == 0:
            # If we checked 5 or more TLS items and none passed, that's suspicious
            print(
                f"Suspicious: TLS server with {total_tls_checked} checked items has 0 compliant items - this suggests a compliance checking issue"
            )
            # This assertion will make the test fail if the issue is detected
            assert False, (
                f"Suspicious: TLS server with {total_tls_checked} checked items has 0 compliant items - this suggests a compliance checking issue"
            )

        if total_ssh_checked >= 3 and total_ssh_passed == 0:
            # If we checked 3 or more SSH items and none passed, that's suspicious
            print(
                f"Suspicious: SSH server with {total_ssh_checked} checked items has 0 compliant items - this suggests a compliance checking issue"
            )
            # This assertion will make the test fail if the issue is detected
            assert False, (
                f"Suspicious: SSH server with {total_ssh_checked} checked items has 0 compliant items - this suggests a compliance checking issue"
            )

    finally:
        # Clean up temporary database
        if os.path.exists(db_path):
            os.unlink(db_path)


def test_compliance_with_database_query_verification():
    """Additional test that verifies compliance results by querying the database directly."""
    import shutil

    template_db = (
        Path(__file__).parent.parent.parent
        / "src"
        / "sslysze_scan"
        / "data"
        / "crypto_standards.db"
    )
    with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as temp_db:
        db_path = temp_db.name
        # Copy the template database to use as our test database
        shutil.copy2(template_db, db_path)

    try:
        # Prepare realistic scan results from fixture data
        scan_results = {}

        # Process SSH scan results (port 22)
        if 22 in SAMPLE_SCAN_DATA["scan_results"]:
            ssh_data = SAMPLE_SCAN_DATA["scan_results"][22]
            scan_results[22] = {
                "kex_algorithms": ssh_data["kex_algorithms"],
                "encryption_algorithms_client_to_server": ssh_data[
                    "encryption_algorithms_client_to_server"
                ],
                "encryption_algorithms_server_to_client": ssh_data[
                    "encryption_algorithms_server_to_client"
                ],
                "mac_algorithms_client_to_server": ssh_data[
                    "mac_algorithms_client_to_server"
                ],
                "mac_algorithms_server_to_client": ssh_data[
                    "mac_algorithms_server_to_client"
                ],
                "host_keys": ssh_data["host_keys"],
            }

        # Process TLS scan results (port 443)
        if 443 in SAMPLE_SCAN_DATA["scan_results"]:
            tls_data = SAMPLE_SCAN_DATA["scan_results"][443]
            scan_results[443] = {
                "tls_versions": tls_data["tls_versions"],
                "cipher_suites": {},
                "supported_groups": tls_data["supported_groups"],
                "certificates": tls_data["certificates"],
            }

            # Add cipher suites by TLS version
            for version, suites in tls_data["cipher_suites"].items():
                scan_results[443]["cipher_suites"][version] = suites

        # Save scan results to database using the regular save function
        scan_start_time = datetime.now(UTC)
        scan_id = write_scan_results(
            db_path,
            SAMPLE_SCAN_DATA["hostname"],
            SAMPLE_SCAN_DATA["ports"],
            scan_results,
            scan_start_time,
            1.0,  # duration
        )

        # Check compliance
        check_compliance(db_path, scan_id)

        # Connect to database to verify compliance entries were created properly
        conn = sqlite3.connect(db_path)
        cursor = conn.cursor()

        # Check that compliance entries were created for the scan
        cursor.execute(
            """
            SELECT check_type, COUNT(*), SUM(CASE WHEN passed = 1 THEN 1 ELSE 0 END)
            FROM scan_compliance_status
            WHERE scan_id = ?
            GROUP BY check_type
        """,
            (scan_id,),
        )

        compliance_counts = cursor.fetchall()

        print("Direct database compliance check:")
        for check_type, total, passed in compliance_counts:
            print(f"  {check_type}: {passed}/{total} compliant")

        # Verify that we have compliance entries for expected check types
        check_types_found = [row[0] for row in compliance_counts]
        expected_check_types = [
            "cipher_suite",
            "supported_group",
            "ssh_kex",
            "ssh_encryption",
            "ssh_mac",
            "ssh_host_key",
        ]

        # At least some of the expected check types should be present
        found_expected = [ct for ct in check_types_found if ct in expected_check_types]
        assert len(found_expected) > 0, (
            f"Expected to find some of {expected_check_types}, but found {found_expected}"
        )

        conn.close()

    finally:
        # Clean up temporary database
        if os.path.exists(db_path):
            os.unlink(db_path)