"""Test for plausible compliance results using realistic scan data from fixtures.""" import os import sqlite3 import tempfile from datetime import UTC, datetime from pathlib import Path from sslysze_scan.db.compliance import check_compliance from sslysze_scan.db.writer import write_scan_results from tests.fixtures.sample_scan_data import SAMPLE_SCAN_DATA def test_compliance_results_with_realistic_scan_data(): """Test that compliance results are plausible when using realistic scan data. This test uses realistic scan data from fixtures to verify that: 1. Servers supporting TLS/SSH connections don't show 0/N compliance results 2. Both compliant and non-compliant items are properly identified 3. The compliance checking logic works with real-world data """ # Use the template database for this test import shutil template_db = ( Path(__file__).parent.parent.parent / "src" / "sslysze_scan" / "data" / "crypto_standards.db" ) with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as temp_db: db_path = temp_db.name # Copy the template database to use as our test database shutil.copy2(template_db, db_path) try: # Prepare realistic scan results from fixture data scan_results = {} # Process SSH scan results (port 22) if 22 in SAMPLE_SCAN_DATA["scan_results"]: ssh_data = SAMPLE_SCAN_DATA["scan_results"][22] scan_results[22] = { "kex_algorithms": ssh_data["kex_algorithms"], "encryption_algorithms_client_to_server": ssh_data[ "encryption_algorithms_client_to_server" ], "encryption_algorithms_server_to_client": ssh_data[ "encryption_algorithms_server_to_client" ], "mac_algorithms_client_to_server": ssh_data[ "mac_algorithms_client_to_server" ], "mac_algorithms_server_to_client": ssh_data[ "mac_algorithms_server_to_client" ], "host_keys": ssh_data["host_keys"], } # Process TLS scan results (port 443) if 443 in SAMPLE_SCAN_DATA["scan_results"]: tls_data = SAMPLE_SCAN_DATA["scan_results"][443] scan_results[443] = { "tls_versions": tls_data["tls_versions"], "cipher_suites": {}, "supported_groups": tls_data["supported_groups"], "certificates": tls_data["certificates"], } # Add cipher suites by TLS version for version, suites in tls_data["cipher_suites"].items(): scan_results[443]["cipher_suites"][version] = suites # Save scan results to database using the regular save function scan_start_time = datetime.now(UTC) scan_id = write_scan_results( db_path, SAMPLE_SCAN_DATA["hostname"], SAMPLE_SCAN_DATA["ports"], scan_results, scan_start_time, 1.0, # duration ) assert scan_id is not None assert scan_id > 0 # Check compliance compliance_results = check_compliance(db_path, scan_id) # Verify basic compliance result structure assert "cipher_suites_checked" in compliance_results assert "cipher_suites_passed" in compliance_results assert "supported_groups_checked" in compliance_results assert "supported_groups_passed" in compliance_results assert "ssh_kex_checked" in compliance_results assert "ssh_kex_passed" in compliance_results assert "ssh_encryption_checked" in compliance_results assert "ssh_encryption_passed" in compliance_results assert "ssh_mac_checked" in compliance_results assert "ssh_mac_passed" in compliance_results assert "ssh_host_keys_checked" in compliance_results assert "ssh_host_keys_passed" in compliance_results # Verify values are non-negative assert compliance_results["cipher_suites_checked"] >= 0 assert compliance_results["cipher_suites_passed"] >= 0 assert compliance_results["supported_groups_checked"] >= 0 assert compliance_results["supported_groups_passed"] >= 0 assert compliance_results["ssh_kex_checked"] >= 0 assert compliance_results["ssh_kex_passed"] >= 0 assert compliance_results["ssh_encryption_checked"] >= 0 assert compliance_results["ssh_encryption_passed"] >= 0 assert compliance_results["ssh_mac_checked"] >= 0 assert compliance_results["ssh_mac_passed"] >= 0 assert compliance_results["ssh_host_keys_checked"] >= 0 assert compliance_results["ssh_host_keys_passed"] >= 0 # Verify that passed count doesn't exceed checked count assert ( compliance_results["cipher_suites_passed"] <= compliance_results["cipher_suites_checked"] ) assert ( compliance_results["supported_groups_passed"] <= compliance_results["supported_groups_checked"] ) assert ( compliance_results["ssh_kex_passed"] <= compliance_results["ssh_kex_checked"] ) assert ( compliance_results["ssh_encryption_passed"] <= compliance_results["ssh_encryption_checked"] ) assert ( compliance_results["ssh_mac_passed"] <= compliance_results["ssh_mac_checked"] ) assert ( compliance_results["ssh_host_keys_passed"] <= compliance_results["ssh_host_keys_checked"] ) # Check that we have meaningful results (not showing implausible 0/N when server supports protocols) # For a server that supports TLS, we should have some cipher suites and groups checked if compliance_results["cipher_suites_checked"] > 0: # Verify the ratio is reasonable (not 0/N when server supports TLS) print( f"Cipher suites: {compliance_results['cipher_suites_passed']}/{compliance_results['cipher_suites_checked']} compliant" ) # Note: We don't enforce a minimum since compliance depends on BSI/IANA standards else: # If no cipher suites were checked, that's acceptable too print("No cipher suites were checked") if compliance_results["supported_groups_checked"] > 0: print( f"Supported groups: {compliance_results['supported_groups_passed']}/{compliance_results['supported_groups_checked']} compliant" ) else: print("No supported groups were checked") # For SSH, we should have some results too if compliance_results["ssh_kex_checked"] > 0: print( f"SSH KEX: {compliance_results['ssh_kex_passed']}/{compliance_results['ssh_kex_checked']} compliant" ) if compliance_results["ssh_encryption_checked"] > 0: print( f"SSH Encryption: {compliance_results['ssh_encryption_passed']}/{compliance_results['ssh_encryption_checked']} compliant" ) if compliance_results["ssh_mac_checked"] > 0: print( f"SSH MAC: {compliance_results['ssh_mac_passed']}/{compliance_results['ssh_mac_checked']} compliant" ) if compliance_results["ssh_host_keys_checked"] > 0: print( f"SSH Host Keys: {compliance_results['ssh_host_keys_passed']}/{compliance_results['ssh_host_keys_checked']} compliant" ) # The main test: ensure that functioning protocols don't show completely non-compliant results # This catches the issue where a server supporting TLS shows 0/N compliance total_tls_checked = ( compliance_results["cipher_suites_checked"] + compliance_results["supported_groups_checked"] ) total_tls_passed = ( compliance_results["cipher_suites_passed"] + compliance_results["supported_groups_passed"] ) total_ssh_checked = ( compliance_results["ssh_kex_checked"] + compliance_results["ssh_encryption_checked"] + compliance_results["ssh_mac_checked"] + compliance_results["ssh_host_keys_checked"] ) total_ssh_passed = ( compliance_results["ssh_kex_passed"] + compliance_results["ssh_encryption_passed"] + compliance_results["ssh_mac_passed"] + compliance_results["ssh_host_keys_passed"] ) # If the server supports TLS and we checked some cipher suites or groups, # there should be a reasonable number of compliant items if total_tls_checked > 0: # Check if we have the problematic 0/N situation (implausible for functioning TLS server) if total_tls_passed == 0: # This would indicate the issue: a functioning TLS server showing 0 compliant items # out of N checked, which is implausible if the server actually supports TLS print( f"WARNING: TLS server with {total_tls_checked} checked items has 0 compliant items" ) # For now, we'll allow this to pass to document the issue, but in a real scenario # we might want to fail the test if we expect at least some compliance # assert total_tls_passed > 0, f"TLS server should have some compliant items, got 0/{total_tls_checked}" # If the server supports SSH and we checked some parameters, # there should be a reasonable number of compliant items if total_ssh_checked > 0: if total_ssh_passed == 0: # This would indicate the issue: a functioning SSH server showing 0 compliant items print( f"WARNING: SSH server with {total_ssh_checked} checked items has 0 compliant items" ) # Same as above, we might want to enforce this in the future # assert total_ssh_passed > 0, f"SSH server should have some compliant items, got 0/{total_ssh_checked}" # More stringent check: if we have a reasonable number of items checked, # we should have at least some minimal compliance # This is a heuristic - for a well-configured server, we'd expect some compliance if total_tls_checked >= 5 and total_tls_passed == 0: # If we checked 5 or more TLS items and none passed, that's suspicious print( f"Suspicious: TLS server with {total_tls_checked} checked items has 0 compliant items - this suggests a compliance checking issue" ) # This assertion will make the test fail if the issue is detected assert False, ( f"Suspicious: TLS server with {total_tls_checked} checked items has 0 compliant items - this suggests a compliance checking issue" ) if total_ssh_checked >= 3 and total_ssh_passed == 0: # If we checked 3 or more SSH items and none passed, that's suspicious print( f"Suspicious: SSH server with {total_ssh_checked} checked items has 0 compliant items - this suggests a compliance checking issue" ) # This assertion will make the test fail if the issue is detected assert False, ( f"Suspicious: SSH server with {total_ssh_checked} checked items has 0 compliant items - this suggests a compliance checking issue" ) finally: # Clean up temporary database if os.path.exists(db_path): os.unlink(db_path) def test_compliance_with_database_query_verification(): """Additional test that verifies compliance results by querying the database directly.""" import shutil template_db = ( Path(__file__).parent.parent.parent / "src" / "sslysze_scan" / "data" / "crypto_standards.db" ) with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as temp_db: db_path = temp_db.name # Copy the template database to use as our test database shutil.copy2(template_db, db_path) try: # Prepare realistic scan results from fixture data scan_results = {} # Process SSH scan results (port 22) if 22 in SAMPLE_SCAN_DATA["scan_results"]: ssh_data = SAMPLE_SCAN_DATA["scan_results"][22] scan_results[22] = { "kex_algorithms": ssh_data["kex_algorithms"], "encryption_algorithms_client_to_server": ssh_data[ "encryption_algorithms_client_to_server" ], "encryption_algorithms_server_to_client": ssh_data[ "encryption_algorithms_server_to_client" ], "mac_algorithms_client_to_server": ssh_data[ "mac_algorithms_client_to_server" ], "mac_algorithms_server_to_client": ssh_data[ "mac_algorithms_server_to_client" ], "host_keys": ssh_data["host_keys"], } # Process TLS scan results (port 443) if 443 in SAMPLE_SCAN_DATA["scan_results"]: tls_data = SAMPLE_SCAN_DATA["scan_results"][443] scan_results[443] = { "tls_versions": tls_data["tls_versions"], "cipher_suites": {}, "supported_groups": tls_data["supported_groups"], "certificates": tls_data["certificates"], } # Add cipher suites by TLS version for version, suites in tls_data["cipher_suites"].items(): scan_results[443]["cipher_suites"][version] = suites # Save scan results to database using the regular save function scan_start_time = datetime.now(UTC) scan_id = write_scan_results( db_path, SAMPLE_SCAN_DATA["hostname"], SAMPLE_SCAN_DATA["ports"], scan_results, scan_start_time, 1.0, # duration ) # Check compliance check_compliance(db_path, scan_id) # Connect to database to verify compliance entries were created properly conn = sqlite3.connect(db_path) cursor = conn.cursor() # Check that compliance entries were created for the scan cursor.execute( """ SELECT check_type, COUNT(*), SUM(CASE WHEN passed = 1 THEN 1 ELSE 0 END) FROM scan_compliance_status WHERE scan_id = ? GROUP BY check_type """, (scan_id,), ) compliance_counts = cursor.fetchall() print("Direct database compliance check:") for check_type, total, passed in compliance_counts: print(f" {check_type}: {passed}/{total} compliant") # Verify that we have compliance entries for expected check types check_types_found = [row[0] for row in compliance_counts] expected_check_types = [ "cipher_suite", "supported_group", "ssh_kex", "ssh_encryption", "ssh_mac", "ssh_host_key", ] # At least some of the expected check types should be present found_expected = [ct for ct in check_types_found if ct in expected_check_types] assert len(found_expected) > 0, ( f"Expected to find some of {expected_check_types}, but found {found_expected}" ) conn.close() finally: # Clean up temporary database if os.path.exists(db_path): os.unlink(db_path)