Add SSH scan support with BSI TR-02102-4 compliance
- SSH scanning via ssh-audit (KEX, encryption, MAC, host keys) - BSI TR-02102-4 and IANA compliance validation for SSH - CSV/Markdown/reST reports for SSH results - Unified compliance schema and database views - Code optimization: modular query/writer architecture
This commit is contained in:
Heiko
370
tests/compliance/test_compliance_with_realistic_data.py
Normal file
370
tests/compliance/test_compliance_with_realistic_data.py
Normal file
@@ -0,0 +1,370 @@
|
||||
"""Test for plausible compliance results using realistic scan data from fixtures."""
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from sslysze_scan.db.compliance import check_compliance
|
||||
from sslysze_scan.db.writer import write_scan_results
|
||||
from tests.fixtures.sample_scan_data import SAMPLE_SCAN_DATA
|
||||
|
||||
|
||||
def test_compliance_results_with_realistic_scan_data():
|
||||
"""Test that compliance results are plausible when using realistic scan data.
|
||||
|
||||
This test uses realistic scan data from fixtures to verify that:
|
||||
1. Servers supporting TLS/SSH connections don't show 0/N compliance results
|
||||
2. Both compliant and non-compliant items are properly identified
|
||||
3. The compliance checking logic works with real-world data
|
||||
"""
|
||||
# Use the template database for this test
|
||||
import shutil
|
||||
|
||||
template_db = (
|
||||
Path(__file__).parent.parent.parent
|
||||
/ "src"
|
||||
/ "sslysze_scan"
|
||||
/ "data"
|
||||
/ "crypto_standards.db"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as temp_db:
|
||||
db_path = temp_db.name
|
||||
# Copy the template database to use as our test database
|
||||
shutil.copy2(template_db, db_path)
|
||||
|
||||
try:
|
||||
# Prepare realistic scan results from fixture data
|
||||
scan_results = {}
|
||||
|
||||
# Process SSH scan results (port 22)
|
||||
if 22 in SAMPLE_SCAN_DATA["scan_results"]:
|
||||
ssh_data = SAMPLE_SCAN_DATA["scan_results"][22]
|
||||
scan_results[22] = {
|
||||
"kex_algorithms": ssh_data["kex_algorithms"],
|
||||
"encryption_algorithms_client_to_server": ssh_data[
|
||||
"encryption_algorithms_client_to_server"
|
||||
],
|
||||
"encryption_algorithms_server_to_client": ssh_data[
|
||||
"encryption_algorithms_server_to_client"
|
||||
],
|
||||
"mac_algorithms_client_to_server": ssh_data[
|
||||
"mac_algorithms_client_to_server"
|
||||
],
|
||||
"mac_algorithms_server_to_client": ssh_data[
|
||||
"mac_algorithms_server_to_client"
|
||||
],
|
||||
"host_keys": ssh_data["host_keys"],
|
||||
}
|
||||
|
||||
# Process TLS scan results (port 443)
|
||||
if 443 in SAMPLE_SCAN_DATA["scan_results"]:
|
||||
tls_data = SAMPLE_SCAN_DATA["scan_results"][443]
|
||||
scan_results[443] = {
|
||||
"tls_versions": tls_data["tls_versions"],
|
||||
"cipher_suites": {},
|
||||
"supported_groups": tls_data["supported_groups"],
|
||||
"certificates": tls_data["certificates"],
|
||||
}
|
||||
|
||||
# Add cipher suites by TLS version
|
||||
for version, suites in tls_data["cipher_suites"].items():
|
||||
scan_results[443]["cipher_suites"][version] = suites
|
||||
|
||||
# Save scan results to database using the regular save function
|
||||
scan_start_time = datetime.now(UTC)
|
||||
scan_id = write_scan_results(
|
||||
db_path,
|
||||
SAMPLE_SCAN_DATA["hostname"],
|
||||
SAMPLE_SCAN_DATA["ports"],
|
||||
scan_results,
|
||||
scan_start_time,
|
||||
1.0, # duration
|
||||
)
|
||||
|
||||
assert scan_id is not None
|
||||
assert scan_id > 0
|
||||
|
||||
# Check compliance
|
||||
compliance_results = check_compliance(db_path, scan_id)
|
||||
|
||||
# Verify basic compliance result structure
|
||||
assert "cipher_suites_checked" in compliance_results
|
||||
assert "cipher_suites_passed" in compliance_results
|
||||
assert "supported_groups_checked" in compliance_results
|
||||
assert "supported_groups_passed" in compliance_results
|
||||
assert "ssh_kex_checked" in compliance_results
|
||||
assert "ssh_kex_passed" in compliance_results
|
||||
assert "ssh_encryption_checked" in compliance_results
|
||||
assert "ssh_encryption_passed" in compliance_results
|
||||
assert "ssh_mac_checked" in compliance_results
|
||||
assert "ssh_mac_passed" in compliance_results
|
||||
assert "ssh_host_keys_checked" in compliance_results
|
||||
assert "ssh_host_keys_passed" in compliance_results
|
||||
|
||||
# Verify values are non-negative
|
||||
assert compliance_results["cipher_suites_checked"] >= 0
|
||||
assert compliance_results["cipher_suites_passed"] >= 0
|
||||
assert compliance_results["supported_groups_checked"] >= 0
|
||||
assert compliance_results["supported_groups_passed"] >= 0
|
||||
assert compliance_results["ssh_kex_checked"] >= 0
|
||||
assert compliance_results["ssh_kex_passed"] >= 0
|
||||
assert compliance_results["ssh_encryption_checked"] >= 0
|
||||
assert compliance_results["ssh_encryption_passed"] >= 0
|
||||
assert compliance_results["ssh_mac_checked"] >= 0
|
||||
assert compliance_results["ssh_mac_passed"] >= 0
|
||||
assert compliance_results["ssh_host_keys_checked"] >= 0
|
||||
assert compliance_results["ssh_host_keys_passed"] >= 0
|
||||
|
||||
# Verify that passed count doesn't exceed checked count
|
||||
assert (
|
||||
compliance_results["cipher_suites_passed"]
|
||||
<= compliance_results["cipher_suites_checked"]
|
||||
)
|
||||
assert (
|
||||
compliance_results["supported_groups_passed"]
|
||||
<= compliance_results["supported_groups_checked"]
|
||||
)
|
||||
assert (
|
||||
compliance_results["ssh_kex_passed"] <= compliance_results["ssh_kex_checked"]
|
||||
)
|
||||
assert (
|
||||
compliance_results["ssh_encryption_passed"]
|
||||
<= compliance_results["ssh_encryption_checked"]
|
||||
)
|
||||
assert (
|
||||
compliance_results["ssh_mac_passed"] <= compliance_results["ssh_mac_checked"]
|
||||
)
|
||||
assert (
|
||||
compliance_results["ssh_host_keys_passed"]
|
||||
<= compliance_results["ssh_host_keys_checked"]
|
||||
)
|
||||
|
||||
# Check that we have meaningful results (not showing implausible 0/N when server supports protocols)
|
||||
# For a server that supports TLS, we should have some cipher suites and groups checked
|
||||
if compliance_results["cipher_suites_checked"] > 0:
|
||||
# Verify the ratio is reasonable (not 0/N when server supports TLS)
|
||||
print(
|
||||
f"Cipher suites: {compliance_results['cipher_suites_passed']}/{compliance_results['cipher_suites_checked']} compliant"
|
||||
)
|
||||
# Note: We don't enforce a minimum since compliance depends on BSI/IANA standards
|
||||
else:
|
||||
# If no cipher suites were checked, that's acceptable too
|
||||
print("No cipher suites were checked")
|
||||
|
||||
if compliance_results["supported_groups_checked"] > 0:
|
||||
print(
|
||||
f"Supported groups: {compliance_results['supported_groups_passed']}/{compliance_results['supported_groups_checked']} compliant"
|
||||
)
|
||||
else:
|
||||
print("No supported groups were checked")
|
||||
|
||||
# For SSH, we should have some results too
|
||||
if compliance_results["ssh_kex_checked"] > 0:
|
||||
print(
|
||||
f"SSH KEX: {compliance_results['ssh_kex_passed']}/{compliance_results['ssh_kex_checked']} compliant"
|
||||
)
|
||||
if compliance_results["ssh_encryption_checked"] > 0:
|
||||
print(
|
||||
f"SSH Encryption: {compliance_results['ssh_encryption_passed']}/{compliance_results['ssh_encryption_checked']} compliant"
|
||||
)
|
||||
if compliance_results["ssh_mac_checked"] > 0:
|
||||
print(
|
||||
f"SSH MAC: {compliance_results['ssh_mac_passed']}/{compliance_results['ssh_mac_checked']} compliant"
|
||||
)
|
||||
if compliance_results["ssh_host_keys_checked"] > 0:
|
||||
print(
|
||||
f"SSH Host Keys: {compliance_results['ssh_host_keys_passed']}/{compliance_results['ssh_host_keys_checked']} compliant"
|
||||
)
|
||||
|
||||
# The main test: ensure that functioning protocols don't show completely non-compliant results
|
||||
# This catches the issue where a server supporting TLS shows 0/N compliance
|
||||
total_tls_checked = (
|
||||
compliance_results["cipher_suites_checked"]
|
||||
+ compliance_results["supported_groups_checked"]
|
||||
)
|
||||
total_tls_passed = (
|
||||
compliance_results["cipher_suites_passed"]
|
||||
+ compliance_results["supported_groups_passed"]
|
||||
)
|
||||
|
||||
total_ssh_checked = (
|
||||
compliance_results["ssh_kex_checked"]
|
||||
+ compliance_results["ssh_encryption_checked"]
|
||||
+ compliance_results["ssh_mac_checked"]
|
||||
+ compliance_results["ssh_host_keys_checked"]
|
||||
)
|
||||
total_ssh_passed = (
|
||||
compliance_results["ssh_kex_passed"]
|
||||
+ compliance_results["ssh_encryption_passed"]
|
||||
+ compliance_results["ssh_mac_passed"]
|
||||
+ compliance_results["ssh_host_keys_passed"]
|
||||
)
|
||||
|
||||
# If the server supports TLS and we checked some cipher suites or groups,
|
||||
# there should be a reasonable number of compliant items
|
||||
if total_tls_checked > 0:
|
||||
# Check if we have the problematic 0/N situation (implausible for functioning TLS server)
|
||||
if total_tls_passed == 0:
|
||||
# This would indicate the issue: a functioning TLS server showing 0 compliant items
|
||||
# out of N checked, which is implausible if the server actually supports TLS
|
||||
print(
|
||||
f"WARNING: TLS server with {total_tls_checked} checked items has 0 compliant items"
|
||||
)
|
||||
# For now, we'll allow this to pass to document the issue, but in a real scenario
|
||||
# we might want to fail the test if we expect at least some compliance
|
||||
# assert total_tls_passed > 0, f"TLS server should have some compliant items, got 0/{total_tls_checked}"
|
||||
|
||||
# If the server supports SSH and we checked some parameters,
|
||||
# there should be a reasonable number of compliant items
|
||||
if total_ssh_checked > 0:
|
||||
if total_ssh_passed == 0:
|
||||
# This would indicate the issue: a functioning SSH server showing 0 compliant items
|
||||
print(
|
||||
f"WARNING: SSH server with {total_ssh_checked} checked items has 0 compliant items"
|
||||
)
|
||||
# Same as above, we might want to enforce this in the future
|
||||
# assert total_ssh_passed > 0, f"SSH server should have some compliant items, got 0/{total_ssh_checked}"
|
||||
|
||||
# More stringent check: if we have a reasonable number of items checked,
|
||||
# we should have at least some minimal compliance
|
||||
# This is a heuristic - for a well-configured server, we'd expect some compliance
|
||||
if total_tls_checked >= 5 and total_tls_passed == 0:
|
||||
# If we checked 5 or more TLS items and none passed, that's suspicious
|
||||
print(
|
||||
f"Suspicious: TLS server with {total_tls_checked} checked items has 0 compliant items - this suggests a compliance checking issue"
|
||||
)
|
||||
# This assertion will make the test fail if the issue is detected
|
||||
assert False, (
|
||||
f"Suspicious: TLS server with {total_tls_checked} checked items has 0 compliant items - this suggests a compliance checking issue"
|
||||
)
|
||||
|
||||
if total_ssh_checked >= 3 and total_ssh_passed == 0:
|
||||
# If we checked 3 or more SSH items and none passed, that's suspicious
|
||||
print(
|
||||
f"Suspicious: SSH server with {total_ssh_checked} checked items has 0 compliant items - this suggests a compliance checking issue"
|
||||
)
|
||||
# This assertion will make the test fail if the issue is detected
|
||||
assert False, (
|
||||
f"Suspicious: SSH server with {total_ssh_checked} checked items has 0 compliant items - this suggests a compliance checking issue"
|
||||
)
|
||||
|
||||
finally:
|
||||
# Clean up temporary database
|
||||
if os.path.exists(db_path):
|
||||
os.unlink(db_path)
|
||||
|
||||
|
||||
def test_compliance_with_database_query_verification():
|
||||
"""Additional test that verifies compliance results by querying the database directly."""
|
||||
import shutil
|
||||
|
||||
template_db = (
|
||||
Path(__file__).parent.parent.parent
|
||||
/ "src"
|
||||
/ "sslysze_scan"
|
||||
/ "data"
|
||||
/ "crypto_standards.db"
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as temp_db:
|
||||
db_path = temp_db.name
|
||||
# Copy the template database to use as our test database
|
||||
shutil.copy2(template_db, db_path)
|
||||
|
||||
try:
|
||||
# Prepare realistic scan results from fixture data
|
||||
scan_results = {}
|
||||
|
||||
# Process SSH scan results (port 22)
|
||||
if 22 in SAMPLE_SCAN_DATA["scan_results"]:
|
||||
ssh_data = SAMPLE_SCAN_DATA["scan_results"][22]
|
||||
scan_results[22] = {
|
||||
"kex_algorithms": ssh_data["kex_algorithms"],
|
||||
"encryption_algorithms_client_to_server": ssh_data[
|
||||
"encryption_algorithms_client_to_server"
|
||||
],
|
||||
"encryption_algorithms_server_to_client": ssh_data[
|
||||
"encryption_algorithms_server_to_client"
|
||||
],
|
||||
"mac_algorithms_client_to_server": ssh_data[
|
||||
"mac_algorithms_client_to_server"
|
||||
],
|
||||
"mac_algorithms_server_to_client": ssh_data[
|
||||
"mac_algorithms_server_to_client"
|
||||
],
|
||||
"host_keys": ssh_data["host_keys"],
|
||||
}
|
||||
|
||||
# Process TLS scan results (port 443)
|
||||
if 443 in SAMPLE_SCAN_DATA["scan_results"]:
|
||||
tls_data = SAMPLE_SCAN_DATA["scan_results"][443]
|
||||
scan_results[443] = {
|
||||
"tls_versions": tls_data["tls_versions"],
|
||||
"cipher_suites": {},
|
||||
"supported_groups": tls_data["supported_groups"],
|
||||
"certificates": tls_data["certificates"],
|
||||
}
|
||||
|
||||
# Add cipher suites by TLS version
|
||||
for version, suites in tls_data["cipher_suites"].items():
|
||||
scan_results[443]["cipher_suites"][version] = suites
|
||||
|
||||
# Save scan results to database using the regular save function
|
||||
scan_start_time = datetime.now(UTC)
|
||||
scan_id = write_scan_results(
|
||||
db_path,
|
||||
SAMPLE_SCAN_DATA["hostname"],
|
||||
SAMPLE_SCAN_DATA["ports"],
|
||||
scan_results,
|
||||
scan_start_time,
|
||||
1.0, # duration
|
||||
)
|
||||
|
||||
# Check compliance
|
||||
check_compliance(db_path, scan_id)
|
||||
|
||||
# Connect to database to verify compliance entries were created properly
|
||||
conn = sqlite3.connect(db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Check that compliance entries were created for the scan
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT check_type, COUNT(*), SUM(CASE WHEN passed = 1 THEN 1 ELSE 0 END)
|
||||
FROM scan_compliance_status
|
||||
WHERE scan_id = ?
|
||||
GROUP BY check_type
|
||||
""",
|
||||
(scan_id,),
|
||||
)
|
||||
|
||||
compliance_counts = cursor.fetchall()
|
||||
|
||||
print("Direct database compliance check:")
|
||||
for check_type, total, passed in compliance_counts:
|
||||
print(f" {check_type}: {passed}/{total} compliant")
|
||||
|
||||
# Verify that we have compliance entries for expected check types
|
||||
check_types_found = [row[0] for row in compliance_counts]
|
||||
expected_check_types = [
|
||||
"cipher_suite",
|
||||
"supported_group",
|
||||
"ssh_kex",
|
||||
"ssh_encryption",
|
||||
"ssh_mac",
|
||||
"ssh_host_key",
|
||||
]
|
||||
|
||||
# At least some of the expected check types should be present
|
||||
found_expected = [ct for ct in check_types_found if ct in expected_check_types]
|
||||
assert len(found_expected) > 0, (
|
||||
f"Expected to find some of {expected_check_types}, but found {found_expected}"
|
||||
)
|
||||
|
||||
conn.close()
|
||||
|
||||
finally:
|
||||
# Clean up temporary database
|
||||
if os.path.exists(db_path):
|
||||
os.unlink(db_path)
|
||||
Reference in New Issue
Block a user