truf-server/tests/test_runtime_security.py

from contextlib import contextmanager
import hashlib
import json
import os
from pathlib import Path
import stat
import subprocess
import sys
import tempfile
import time
import unittest
from unittest import mock


ROOT = Path(__file__).resolve().parents[1]
APP_DIR = ROOT / 'app'
sys.path.insert(0, str(APP_DIR))

import runtime_security


@contextmanager
def isolated_endpoint_authority(directory):
    root = os.path.join(runtime_security.canonical_path(directory), 'endpoint-authority')
    namespace = hashlib.sha256(os.fsencode(root)).hexdigest()
    mutex_name = runtime_security.cluster_endpoint_mutex_name
    # Relocate only authority names; real private-file locks and Windows mutexes remain.
    with mock.patch.object(runtime_security, '_cluster_endpoint_lock_root', return_value=root), \
            mock.patch.object(runtime_security, 'cluster_endpoint_mutex_name', side_effect=lambda *args, **kwargs: (
                mutex_name(*args, **kwargs) + '.fixture-' + namespace
            )):
        yield root


class RuntimeSecurityTests(unittest.TestCase):
    def test_atomic_directory_publication_is_nonreplacing_and_preserves_tree(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            source_parent = os.path.join(temp_dir, 'active')
            destination_parent = os.path.join(temp_dir, 'abandoned')
            os.makedirs(source_parent)
            os.makedirs(destination_parent)
            source = os.path.join(source_parent, 'assignment')
            destination = os.path.join(destination_parent, 'assignment')
            os.makedirs(source)
            Path(source, 'proof.txt').write_text('proof', encoding='ascii')
            runtime_security.durable_publish_directory(source, destination)
            self.assertFalse(os.path.exists(source))
            self.assertEqual(
                Path(destination, 'proof.txt').read_text(encoding='ascii'),
                'proof',
            )

            replacement = os.path.join(source_parent, 'assignment')
            os.makedirs(replacement)
            with self.assertRaises(FileExistsError):
                runtime_security.durable_publish_directory(
                    replacement, destination,
                )
            self.assertTrue(os.path.isdir(replacement))

    @unittest.skipUnless(os.name != 'nt' and os.geteuid() == 0, 'requires POSIX root')
    def test_stable_root_file_requires_public_immutable_metadata(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            manifest_root = os.path.join(temp_dir, 'manifests')
            package_root = os.path.join(manifest_root, 'packages')
            os.makedirs(package_root)
            path = os.path.join(package_root, 'worker-package.json')
            with open(path, 'wb') as handle:
                handle.write(b'fixture')
            os.chmod(path, 0o644)
            self.assertEqual(
                runtime_security.read_stable_root_file(path, 7, manifest_root),
                b'fixture',
            )

            os.chmod(path, 0o600)
            with self.assertRaises(runtime_security.PrivateFileError):
                runtime_security.read_stable_root_file(path, 7, manifest_root)

            os.chmod(path, 0o644)
            linked = os.path.join(temp_dir, 'linked.json')
            os.link(path, linked)
            with self.assertRaises(runtime_security.PrivateFileError):
                runtime_security.read_stable_root_file(path, 7, manifest_root)

            os.unlink(linked)
            os.chmod(package_root, 0o777)
            with self.assertRaises(runtime_security.PrivateFileError):
                runtime_security.read_stable_root_file(path, 7, manifest_root)

    def test_discovery_preflight_keeps_code_authority_without_scanner_native_tools(self):
        config = {
            'global': {
                'project_dir': 'project',
                'trufflehog_path': 'trufflehog-fixture',
                'trufflehog_config': 'detectors.yaml',
                'secrets_file': 'secrets.yaml',
                'proxy_file': 'proxies.txt',
            },
            'sources': {
                'gitlab': {'trufflehog_config': 'gitlab-detectors.yaml'},
            },
        }
        with mock.patch.object(runtime_security, 'require_private_file') as private_file, \
                mock.patch.object(runtime_security, 'require_private_directory'), \
                mock.patch('lifecycle_authority.manifest_authority_paths',
                           return_value=['project/authority.py']) as manifest, \
                mock.patch('lifecycle_authority.resolve_manifest_executable') as resolve, \
                mock.patch.object(runtime_security, 'require_trusted_native_executable') as native:
            self.assertTrue(runtime_security.preflight_lifecycle_paths(
                'config.yaml', config, authority_profile='discovery-producer',
            ))

        manifest.assert_called_once_with(
            'project', 'trufflehog-fixture', policy_paths=[],
            existing_only=True, include_executables=False,
        )
        resolve.assert_not_called()
        native.assert_not_called()
        private_file.assert_any_call('config.yaml')
        private_file.assert_any_call('project/authority.py')
        self.assertNotIn(mock.call('detectors.yaml'), private_file.call_args_list)
        self.assertNotIn(mock.call('gitlab-detectors.yaml'), private_file.call_args_list)

    def test_unknown_lifecycle_preflight_profile_fails_closed(self):
        with self.assertRaisesRegex(runtime_security.PrivateFileError, 'unsupported'):
            runtime_security.preflight_lifecycle_paths(
                'config.yaml', {}, authority_profile='arbitrary',
            )

    def test_private_json_extended_bound_is_explicit_and_capped(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            directory = os.path.join(temp_dir, 'private')
            runtime_security.ensure_private_directory(directory, reject_reparse=True)
            path = os.path.join(directory, 'manifest.json')
            value = {'payload': 'x' * (runtime_security.MAX_PRIVATE_JSON_BYTES + 1)}
            with self.assertRaisesRegex(ValueError, 'too large'):
                runtime_security.atomic_write_private_json(path, value)
            runtime_security.atomic_write_private_json(
                path, value, max_bytes=runtime_security.MAX_EXTENDED_PRIVATE_JSON_BYTES,
            )
            self.assertEqual(
                runtime_security.read_private_json(
                    path, max_bytes=runtime_security.MAX_EXTENDED_PRIVATE_JSON_BYTES,
                ),
                value,
            )
            with self.assertRaisesRegex(ValueError, 'bound is invalid'):
                runtime_security.atomic_write_private_json(
                    path, value,
                    max_bytes=runtime_security.MAX_EXTENDED_PRIVATE_JSON_BYTES + 1,
                )

    def test_cluster_authority_create_parent_hardens_existing_directory(self):
        with tempfile.TemporaryDirectory() as temp_dir, isolated_endpoint_authority(temp_dir) as endpoint_root:
            runtime_security.ensure_private_directory(temp_dir, reject_reparse=True)
            runtime = os.path.join(temp_dir, 'runtime')
            postgres = os.path.join(runtime, 'postgres')
            os.makedirs(postgres)
            if os.name != 'nt':
                os.chmod(postgres, 0o755)
                self.assertFalse(runtime_security.private_directory_ready(postgres))
            config = {'global': {'runtime_dir': runtime}}
            lock = runtime_security.ClusterAuthorityLock(
                config,
                create_parent=True,
                endpoint_dsn='postgresql://truf:fixture@127.0.0.1:15432/truf',
            )
            self.assertEqual(os.path.dirname(lock.endpoint_path), endpoint_root)
            with mock.patch.object(runtime_security, 'harden_private_directory',
                                   wraps=runtime_security.harden_private_directory) as harden:
                try:
                    lock.acquire()
                    self.assertTrue(lock.acquired)
                finally:
                    lock.release()
            harden.assert_any_call(runtime_security.canonical_path(postgres))
            self.assertFalse(lock.acquired)
            self.assertTrue(runtime_security.private_directory_ready(postgres))
            if os.name != 'nt':
                self.assertTrue(runtime_security.private_directory_ready(endpoint_root))

    def test_directory_and_file_exact_private_allowlists_verify(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            directory = os.path.join(temp_dir, 'private')
            runtime_security.ensure_private_directory(directory, reject_reparse=True)
            path = os.path.join(directory, 'secret.txt')
            Path(path).write_text('secret', encoding='ascii')
            runtime_security.harden_private_file(path)
            self.assertTrue(runtime_security.private_directory_ready(directory))
            self.assertTrue(runtime_security.private_file_ready(path))
            if os.name == 'nt':
                sddl = runtime_security._windows_private_sddl(directory).upper()
                self.assertEqual(sddl.count('(A;OICI;FA;'), 3)
            else:
                self.assertEqual(stat.S_IMODE(os.stat(directory).st_mode), 0o700)
                self.assertEqual(stat.S_IMODE(os.stat(path).st_mode), 0o600)

    def test_hardening_rejects_reparse_parent_without_touching_external_file(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            external = os.path.join(temp_dir, 'external')
            os.makedirs(external)
            target = os.path.join(external, 'target.txt')
            Path(target).write_text('external', encoding='ascii')
            before = stat.S_IMODE(os.stat(target).st_mode)
            link = os.path.join(temp_dir, 'linked')
            try:
                os.symlink(external, link, target_is_directory=True)
            except (OSError, NotImplementedError):
                self.skipTest('directory symlink creation is unavailable')
            with self.assertRaises(runtime_security.PrivateFileError):
                runtime_security.harden_private_file(os.path.join(link, 'target.txt'))
            self.assertEqual(stat.S_IMODE(os.stat(target).st_mode), before)

    def test_sensitive_startup_paths_fail_instead_of_repairing_existing_public_directory(self):
        if os.name == 'nt':
            self.skipTest('portable public-mode fixture is POSIX-specific')
        with tempfile.TemporaryDirectory() as temp_dir:
            public = os.path.join(temp_dir, 'runtime')
            os.makedirs(public, mode=0o755)
            os.chmod(public, 0o755)
            with self.assertRaises(runtime_security.PrivateFileError):
                runtime_security.require_sensitive_runtime_paths({'runtime_dir': public}, create=True)

    def test_sensitive_file_accepts_protected_root_owned_parent(self):
        details = mock.Mock(
            st_mode=stat.S_IFDIR | 0o755,
            st_uid=0,
            st_gid=0,
        )
        with mock.patch.object(runtime_security.os, 'name', 'posix'), \
                mock.patch.object(runtime_security, 'reject_reparse_components',
                                  return_value='/data/config'), \
                mock.patch.object(runtime_security, 'private_directory_ready',
                                  return_value=False), \
                mock.patch.object(runtime_security.os, 'stat', return_value=details), \
                mock.patch.object(runtime_security.os, 'geteuid', return_value=10001,
                                  create=True), \
                mock.patch.object(runtime_security.os, 'getegid', return_value=10001,
                                  create=True), \
                mock.patch.object(runtime_security.os, 'getgroups', return_value=[],
                                  create=True):
            self.assertEqual(
                runtime_security.require_protected_sensitive_file_parent('/data/config'),
                '/data/config',
            )

    @mock.patch.object(runtime_security.os, 'name', 'posix')
    @mock.patch.object(runtime_security, 'private_directory_ready', return_value=False)
    @mock.patch.object(runtime_security.os, 'geteuid', return_value=10001, create=True)
    @mock.patch.object(runtime_security.os, 'getegid', return_value=10001, create=True)
    @mock.patch.object(runtime_security.os, 'getgroups', return_value=[], create=True)
    def test_sensitive_file_rejects_unprotected_root_parent(
        self, _groups, _egid, _euid, _private,
    ):
        fixtures = {
            'group-writable': (0, 0, 0o775),
            'other-writable': (0, 0, 0o757),
            'inaccessible': (0, 0, 0o750),
            'runtime-owned-public': (10001, 10001, 0o755),
        }
        for name, (uid, gid, mode) in fixtures.items():
            with self.subTest(name=name), \
                    mock.patch.object(runtime_security, 'reject_reparse_components',
                                      return_value='/data/config'), \
                    mock.patch.object(runtime_security.os, 'stat', return_value=mock.Mock(
                        st_mode=stat.S_IFDIR | mode, st_uid=uid, st_gid=gid,
                    )):
                with self.assertRaises(runtime_security.PrivateFileError):
                    runtime_security.require_protected_sensitive_file_parent('/data/config')

    def test_sensitive_file_parent_rejects_link_before_metadata_check(self):
        with mock.patch.object(
            runtime_security, 'reject_reparse_components',
            side_effect=runtime_security.PrivateFileError('link'),
        ), mock.patch.object(runtime_security.os, 'stat') as inspect:
            with self.assertRaises(runtime_security.PrivateFileError):
                runtime_security.require_protected_sensitive_file_parent('/data/config')
        inspect.assert_not_called()

    @mock.patch.object(runtime_security.os, 'name', 'posix')
    def test_private_file_rejects_permissive_or_wrong_owner(self):
        for name, uid, mode in (
            ('permissive', 10001, 0o640),
            ('wrong-owner', 0, 0o600),
        ):
            with self.subTest(name=name), \
                    mock.patch.object(runtime_security, 'reject_reparse_components',
                                      return_value='/data/config/secrets.yaml'), \
                    mock.patch.object(runtime_security.os, 'stat', return_value=mock.Mock(
                        st_mode=stat.S_IFREG | mode, st_uid=uid, st_gid=10001,
                    )), \
                    mock.patch.object(runtime_security.os, 'geteuid', return_value=10001,
                                      create=True):
                self.assertFalse(
                    runtime_security.private_file_ready('/data/config/secrets.yaml')
                )

    def test_windows_acl_verification_requires_actual_owner_sid_not_owner_rights(self):
        with tempfile.TemporaryDirectory() as temp_dir, \
                mock.patch.object(runtime_security.os, 'name', 'nt'), \
                mock.patch.object(runtime_security, 'reject_reparse_components'), \
                mock.patch.object(runtime_security, '_windows_current_user_sid', return_value='S-1-5-21-100'):
            valid = (
                'O:S-1-5-21-100D:P'
                '(A;OICI;FA;;;S-1-5-21-100)(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)'
            )
            with mock.patch.object(runtime_security, '_windows_private_sddl', return_value=valid):
                self.assertTrue(runtime_security._private_acl_ready(temp_dir, directory=True))
            generic_owner_rights = 'O:S-1-5-21-100D:P(A;OICI;FA;;;OW)(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)'
            with mock.patch.object(runtime_security, '_windows_private_sddl', return_value=generic_owner_rights):
                self.assertFalse(runtime_security._private_acl_ready(temp_dir, directory=True))
            foreign_owner = valid.replace('O:S-1-5-21-100', 'O:S-1-5-21-999', 1)
            with mock.patch.object(runtime_security, '_windows_private_sddl', return_value=foreign_owner):
                self.assertFalse(runtime_security._private_acl_ready(temp_dir, directory=True))

    def test_lifecycle_preflight_rejects_reparse_work_path_without_mutation(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            runtime_security.ensure_private_directory(temp_dir, reject_reparse=True)
            project = os.path.join(temp_dir, 'project')
            runtime = os.path.join(temp_dir, 'runtime')
            external = os.path.join(temp_dir, 'external')
            for path in (
                project, runtime, external,
                os.path.join(runtime, 'control'), os.path.join(runtime, 'logs'),
                os.path.join(runtime, 'results'), os.path.join(runtime, 'queues'),
                os.path.join(runtime, 'state'), os.path.join(runtime, 'keychecks'),
                os.path.join(runtime, 'result_spool'), os.path.join(runtime, 'postgres'),
            ):
                runtime_security.ensure_private_directory(path, reject_reparse=True)
            linked_work = os.path.join(temp_dir, 'work')
            try:
                os.symlink(external, linked_work, target_is_directory=True)
            except (OSError, NotImplementedError):
                self.skipTest('directory symlink creation is unavailable')
            config_path = os.path.join(project, 'config.yaml')
            Path(config_path).write_text('{}\n', encoding='ascii')
            runtime_security.harden_private_file(config_path)
            config = {
                'global': {
                    'root_dir': temp_dir,
                    'project_dir': project,
                    'runtime_dir': runtime,
                    'control_dir': os.path.join(runtime, 'control'),
                    'log_dir': os.path.join(runtime, 'logs'),
                    'work_dir': linked_work,
                    'results_dir': os.path.join(runtime, 'results'),
                    'queue_dir': os.path.join(runtime, 'queues'),
                    'state_dir': os.path.join(runtime, 'state'),
                    'keycheck_dir': os.path.join(runtime, 'keychecks'),
                    'result_spool_dir': os.path.join(runtime, 'result_spool'),
                },
                'supervisor': {
                    'control_dir': os.path.join(runtime, 'control'),
                    'log_dir': os.path.join(runtime, 'logs'),
                },
            }
            before = os.stat(external, follow_symlinks=False)
            with self.assertRaisesRegex(runtime_security.PrivateFileError, 'offline hardening'):
                runtime_security.preflight_lifecycle_paths(config_path, config)
            after = os.stat(external, follow_symlinks=False)
            self.assertEqual((before.st_mode, before.st_mtime_ns), (after.st_mode, after.st_mtime_ns))

    def test_external_postgres_data_is_bound_to_preflight_and_cluster_lock(self):
        with tempfile.TemporaryDirectory() as temp_dir, isolated_endpoint_authority(temp_dir) as endpoint_root:
            runtime_security.ensure_private_directory(temp_dir, reject_reparse=True)
            project = os.path.join(temp_dir, 'project')
            runtime = os.path.join(temp_dir, 'runtime')
            postgres_dir = os.path.join(runtime, 'postgres')
            external_data = os.path.join(temp_dir, 'external-data')
            for path in (project, runtime, postgres_dir, external_data, endpoint_root):
                runtime_security.ensure_private_directory(path, reject_reparse=True)
            config_path = os.path.join(project, 'config.yaml')
            code_path = os.path.join(project, 'authority.py')
            policy_path = os.path.join(project, 'policy.yaml')
            for path in (config_path, code_path, policy_path):
                Path(path).write_text('{}\n', encoding='ascii')
                runtime_security.harden_private_file(path)
            tools = {name: os.path.join(temp_dir, 'fixture-' + name) for name in ('trufflehog', 'git')}
            config = {'global': {
                'root_dir': temp_dir,
                'project_dir': project,
                'runtime_dir': runtime,
                'postgres_data_dir': external_data,
                'trufflehog_path': tools['trufflehog'],
                'trufflehog_config': policy_path,
            }}

            def resolve(value, *, name, app_dir):
                self.assertEqual(app_dir, project)
                self.assertEqual(value, tools[name] if name == 'trufflehog' else None)
                return tools[name]

            rejected_tool = None

            def verify_native(path):
                self.assertIn(path, tools.values())
                if path == rejected_tool:
                    raise runtime_security.PrivateFileError('fixture native executable is untrusted')
                return path

            with mock.patch('lifecycle_authority.manifest_authority_paths',
                            return_value=[code_path, policy_path]) as manifest, \
                    mock.patch('lifecycle_authority.resolve_manifest_executable', side_effect=resolve) as resolved, \
                    mock.patch.object(runtime_security, 'require_trusted_native_executable', side_effect=verify_native) as native, \
                    mock.patch.object(runtime_security, 'require_private_directory',
                                      wraps=runtime_security.require_private_directory) as directories, \
                    mock.patch.object(runtime_security, 'require_private_file',
                                      wraps=runtime_security.require_private_file) as private_files, \
                    mock.patch.object(runtime_security, 'harden_private_directory',
                                      side_effect=AssertionError('preflight must not repair directories')), \
                    mock.patch.object(runtime_security, 'harden_private_file',
                                      side_effect=AssertionError('preflight must not repair files')):
                self.assertTrue(runtime_security.preflight_lifecycle_paths(config_path, config))
                manifest.assert_called_once_with(
                    project, tools['trufflehog'], policy_paths=[policy_path],
                    existing_only=True, include_executables=False,
                )
                self.assertEqual(resolved.call_args_list, [
                    mock.call(tools['trufflehog'], name='trufflehog', app_dir=project),
                    mock.call(None, name='git', app_dir=project),
                ])
                self.assertEqual(native.call_args_list, [mock.call(path) for path in tools.values()])
                directories.assert_any_call(runtime_security.canonical_path(external_data), create=False)
                if os.name != 'nt':
                    directories.assert_any_call(endpoint_root, create=False)
                for path in (config_path, code_path, policy_path):
                    private_files.assert_any_call(path)

                for rejected_tool in tools.values():
                    with self.subTest(untrusted_tool=os.path.basename(rejected_tool)), \
                            self.assertRaisesRegex(runtime_security.PrivateFileError, 'native executable is untrusted'):
                        runtime_security.preflight_lifecycle_paths(config_path, config)
                rejected_tool = None
                native.reset_mock()
                Path(external_data).rmdir()
                Path(external_data).write_text('not a directory\n', encoding='ascii')
                with self.assertRaises(runtime_security.PrivateFileError):
                    runtime_security.preflight_lifecycle_paths(config_path, config)
                native.assert_not_called()
            self.assertEqual(
                runtime_security.canonical_cluster_data_directory(config),
                runtime_security.canonical_path(external_data),
            )
            self.assertEqual(
                os.path.dirname(runtime_security.cluster_authority_lock_path(config)),
                runtime_security.canonical_path(postgres_dir),
            )
            self.assertNotEqual(
                runtime_security.cluster_authority_lock_path(config),
                runtime_security.cluster_authority_lock_path({
                    'global': {'runtime_dir': runtime},
                }),
            )

    def test_postgres_data_directory_rejects_filesystem_root(self):
        config = {'global': {
            'runtime_dir': os.path.join(os.getcwd(), 'runtime'),
            'postgres_data_dir': os.path.abspath(os.sep),
        }}
        with self.assertRaisesRegex(runtime_security.PrivateFileError, 'volume root|filesystem'):
            runtime_security.canonical_cluster_data_directory(config)

    def test_postgres_data_directory_rejects_reparse_component(self):
        with tempfile.TemporaryDirectory() as temp_dir:
            external = os.path.join(temp_dir, 'external')
            os.makedirs(external)
            linked = os.path.join(temp_dir, 'linked-data')
            try:
                os.symlink(external, linked, target_is_directory=True)
            except (OSError, NotImplementedError):
                self.skipTest('directory symlink creation is unavailable')
            config = {'global': {
                'runtime_dir': os.path.join(temp_dir, 'runtime'),
                'postgres_data_dir': linked,
            }}
            with self.assertRaisesRegex(runtime_security.PrivateFileError, 'reparse point|link'):
                runtime_security.canonical_cluster_data_directory(config)

    def test_cluster_authority_lock_contends_across_alternate_configs_and_processes(self):
        with tempfile.TemporaryDirectory() as temp_dir, isolated_endpoint_authority(temp_dir) as endpoint_root:
            runtime_security.ensure_private_directory(temp_dir, reject_reparse=True)
            runtime_a = os.path.join(temp_dir, 'cluster-a')
            runtime_b = os.path.join(temp_dir, 'cluster-b')
            for runtime in (runtime_a, runtime_b):
                runtime_security.ensure_private_directory(runtime, reject_reparse=True)
                runtime_security.ensure_private_directory(os.path.join(runtime, 'postgres'), reject_reparse=True)
            first = {
                'global': {
                    'runtime_dir': runtime_a,
                    'control_dir': os.path.join(temp_dir, 'control-one'),
                    'database_url': 'postgresql://cluster:one@127.0.0.1:15432/cluster_a',
                },
                'supervisor': {'control_port': 11001},
            }
            alternate = {
                'global': {
                    'runtime_dir': runtime_a,
                    'control_dir': os.path.join(temp_dir, 'control-two'),
                    'database_url': 'postgresql://cluster:different@localhost:15432/cluster_a',
                },
                'supervisor': {'control_port': 22002},
            }
            independent = {
                'global': {
                    'runtime_dir': runtime_b,
                    'control_dir': os.path.join(temp_dir, 'control-three'),
                    'database_url': 'postgresql://cluster:two@127.0.0.1:25432/cluster_b',
                },
                'supervisor': {'control_port': 33003},
            }
            same_endpoint_other_data = {
                'global': {
                    'runtime_dir': runtime_b,
                    'control_dir': os.path.join(temp_dir, 'control-four'),
                    'database_url': 'postgresql://cluster:other-password@127.0.0.1:15432/cluster_a',
                },
                'supervisor': {'control_port': 44004},
            }
            self.assertEqual(
                runtime_security.cluster_authority_lock_path(first),
                runtime_security.cluster_authority_lock_path(alternate),
            )
            self.assertNotEqual(
                runtime_security.cluster_authority_lock_path(first),
                runtime_security.cluster_authority_lock_path(independent),
            )
            self.assertNotEqual(
                runtime_security.cluster_authority_lock_path(first),
                runtime_security.cluster_authority_lock_path(same_endpoint_other_data),
            )
            self.assertEqual(
                runtime_security.cluster_endpoint_authority_lock_path(first, endpoint_dsn=first['global']['database_url']),
                runtime_security.cluster_endpoint_authority_lock_path(same_endpoint_other_data, endpoint_dsn=same_endpoint_other_data['global']['database_url']),
            )
            self.assertNotEqual(
                runtime_security.cluster_endpoint_authority_lock_path(first, endpoint_dsn=first['global']['database_url']),
                runtime_security.cluster_endpoint_authority_lock_path(independent, endpoint_dsn=independent['global']['database_url']),
            )

            fixture_code = (
                'import json,os,runpy,sys\n'
                'fixture=runpy.run_path(sys.argv[1])\n'
                'cfg=json.loads(sys.argv[2])\n'
                'with fixture["isolated_endpoint_authority"](sys.argv[3]) as root:\n'
                '    lock=fixture["runtime_security"].ClusterAuthorityLock(cfg,endpoint_dsn=cfg["global"]["database_url"])\n'
                '    assert os.path.dirname(lock.endpoint_path)==root\n'
            )
            holder_code = fixture_code + (
                '    with lock:\n'
                '        with open(sys.argv[4], "x", encoding="ascii") as ready: ready.write("READY")\n'
                '        sys.stdin.readline()\n'
            )
            contender_code = fixture_code + (
                '    try: lock.acquire()\n'
                '    except BlockingIOError: raise SystemExit(3)\n'
                '    lock.release()\n'
            )
            command = [sys.executable, '-I', '-S', '-B', '-c']
            fixture_path = str(Path(__file__).resolve())
            ready = Path(temp_dir) / 'holder.ready'
            holder = subprocess.Popen(
                command + [holder_code, fixture_path, json.dumps(first), temp_dir, str(ready)],
                stdin=subprocess.PIPE,
                stdout=subprocess.DEVNULL,
                stderr=subprocess.PIPE,
                text=True,
            )
            try:
                deadline = time.monotonic() + 10
                while not ready.exists() and holder.poll() is None and time.monotonic() < deadline:
                    time.sleep(0.02)
                self.assertTrue(ready.exists(), 'isolated lock holder did not become ready')
                blocked = subprocess.run(
                    command + [contender_code, fixture_path, json.dumps(alternate), temp_dir],
                    stdin=subprocess.DEVNULL,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    text=True,
                    timeout=15,
                    check=False,
                )
                self.assertEqual(blocked.returncode, 3, blocked.stderr)
                endpoint_blocked = subprocess.run(
                    command + [contender_code, fixture_path, json.dumps(same_endpoint_other_data), temp_dir],
                    stdin=subprocess.DEVNULL,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    text=True,
                    timeout=15,
                    check=False,
                )
                self.assertEqual(endpoint_blocked.returncode, 3, endpoint_blocked.stderr)
                coexisting = subprocess.run(
                    command + [contender_code, fixture_path, json.dumps(independent), temp_dir],
                    stdin=subprocess.DEVNULL,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    text=True,
                    timeout=15,
                    check=False,
                )
                self.assertEqual(coexisting.returncode, 0, coexisting.stderr)
            finally:
                try:
                    holder.communicate('\n', timeout=10)
                except subprocess.TimeoutExpired:
                    holder.kill()
                    holder.communicate(timeout=10)
            self.assertEqual(holder.returncode, 0)

            maintenance = runtime_security.ClusterAuthorityLock(
                alternate, endpoint_dsn=alternate['global']['database_url'],
            )
            self.assertEqual(os.path.dirname(maintenance.endpoint_path), endpoint_root)
            try:
                maintenance.acquire()
                self.assertTrue(maintenance.acquired)
                runtime_blocked = subprocess.run(
                    command + [contender_code, fixture_path, json.dumps(first), temp_dir],
                    stdin=subprocess.DEVNULL,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    text=True,
                    timeout=15,
                    check=False,
                )
                self.assertEqual(runtime_blocked.returncode, 3, runtime_blocked.stderr)
            finally:
                maintenance.release()


if __name__ == '__main__':
    unittest.main()