sashatrask
850 lines
36 KiB
Python
850 lines
36 KiB
Python
import copy
|
|
from contextlib import contextmanager, ExitStack
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
import stat
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
import uuid
|
|
from unittest import mock
|
|
|
|
import yaml
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
APP_DIR = ROOT / 'app'
|
|
sys.path.insert(0, str(APP_DIR))
|
|
|
|
import host_agent_apply
|
|
from host_agent_protocol import HostAgentAction, HostAgentRequest
|
|
import runtime_document
|
|
|
|
|
|
_REAL_PRIVATE_FILE_LOCK = host_agent_apply.PrivateFileLock
|
|
|
|
|
|
class _Connection:
|
|
is_postgres = True
|
|
|
|
|
|
class _Database:
|
|
def __init__(self):
|
|
self.conn = _Connection()
|
|
self.claims = []
|
|
self.failure = None
|
|
self.replayed = False
|
|
|
|
def claim_runtime_operation_execution(self, **values):
|
|
self.claims.append(values)
|
|
if self.failure is not None:
|
|
raise self.failure
|
|
return {'status': 'running', 'replayed': self.replayed}
|
|
|
|
|
|
class _TestLock:
|
|
held = set()
|
|
|
|
def __init__(self, path):
|
|
self.path = os.fspath(path)
|
|
self.acquired = False
|
|
|
|
def acquire(self):
|
|
if self.path in self.held:
|
|
raise BlockingIOError('held')
|
|
self.held.add(self.path)
|
|
self.acquired = True
|
|
return self
|
|
|
|
def release(self):
|
|
if self.acquired:
|
|
self.held.remove(self.path)
|
|
self.acquired = False
|
|
|
|
|
|
class HostAgentApplyTests(unittest.TestCase):
|
|
@staticmethod
|
|
def proof(request):
|
|
return host_agent_apply._new_stopped_runtime_proof(request.operation_id)
|
|
|
|
@staticmethod
|
|
def rollback_proof(request):
|
|
return host_agent_apply._new_stopped_runtime_proof(
|
|
request.operation_id, purpose='rollback',
|
|
)
|
|
|
|
def documents(self):
|
|
template_payload = (APP_DIR / 'config.linux.yaml').read_bytes()
|
|
template = runtime_document.load_yaml_document(
|
|
template_payload,
|
|
max_bytes=runtime_document.MAX_CONFIG_DOCUMENT_BYTES,
|
|
)
|
|
config = copy.deepcopy(template)
|
|
docker_pool = config['sources']['dockerhub']['auth_pool']
|
|
pools = {
|
|
source['auth_pool']
|
|
for source in config['sources'].values()
|
|
if source.get('auth_pool')
|
|
}
|
|
secrets = {'auth_pools': {
|
|
pool: [{
|
|
'name': pool + '_1',
|
|
**({'username': 'fixture-user'} if pool == docker_pool else {}),
|
|
'token': 'fixture-token',
|
|
}]
|
|
for pool in pools
|
|
}}
|
|
return template_payload, config, secrets
|
|
|
|
@staticmethod
|
|
def write_private(path, payload):
|
|
path.write_bytes(payload)
|
|
os.chmod(path, 0o600)
|
|
|
|
@contextmanager
|
|
def store(
|
|
self, *, candidate_config=True, candidate_secrets=True, real_lock=False,
|
|
):
|
|
template_payload, config, secrets = self.documents()
|
|
active_config = yaml.safe_dump(config).encode('utf-8')
|
|
active_secrets = yaml.safe_dump(secrets).encode('utf-8')
|
|
proposed_config = active_config + b'# candidate config\n'
|
|
proposed_secrets = active_secrets + b'# candidate secrets\n'
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
root = Path(temporary)
|
|
active = root / 'active'
|
|
candidates = root / 'candidates'
|
|
apply = root / 'apply'
|
|
backups = root / 'backups'
|
|
for directory in (active, candidates, apply, backups):
|
|
directory.mkdir(mode=0o700)
|
|
os.chmod(directory, 0o700)
|
|
active_config_path = active / 'config.yaml'
|
|
active_secrets_path = active / 'secrets.yaml'
|
|
candidate_config_path = candidates / 'config.yaml'
|
|
candidate_secrets_path = candidates / 'secrets.yaml'
|
|
template_path = root / 'template.yaml'
|
|
self.write_private(active_config_path, active_config)
|
|
self.write_private(active_secrets_path, active_secrets)
|
|
self.write_private(template_path, template_payload)
|
|
if candidate_config:
|
|
self.write_private(candidate_config_path, proposed_config)
|
|
if candidate_secrets:
|
|
self.write_private(candidate_secrets_path, proposed_secrets)
|
|
details = os.stat(root)
|
|
uid = getattr(details, 'st_uid', 0)
|
|
gid = getattr(details, 'st_gid', 0)
|
|
runtime_uid = (
|
|
10001
|
|
if os.name != 'nt' and os.geteuid() == 0
|
|
else uid
|
|
)
|
|
runtime_gid = (
|
|
10001
|
|
if os.name != 'nt' and os.geteuid() == 0
|
|
else gid
|
|
)
|
|
if os.name != 'nt' and runtime_uid != uid:
|
|
os.chown(candidates, runtime_uid, runtime_gid)
|
|
for path in (
|
|
active_config_path,
|
|
active_secrets_path,
|
|
candidate_config_path,
|
|
candidate_secrets_path,
|
|
):
|
|
if path.exists():
|
|
os.chown(path, runtime_uid, runtime_gid)
|
|
values = {
|
|
'root': root,
|
|
'active': active,
|
|
'candidates': candidates,
|
|
'backups': backups,
|
|
'active_config_path': active_config_path,
|
|
'active_secrets_path': active_secrets_path,
|
|
'candidate_config_path': candidate_config_path,
|
|
'candidate_secrets_path': candidate_secrets_path,
|
|
'active_config': active_config,
|
|
'active_secrets': active_secrets,
|
|
'proposed_config': proposed_config,
|
|
'proposed_secrets': proposed_secrets,
|
|
}
|
|
with ExitStack() as stack:
|
|
stack.enter_context(mock.patch.multiple(
|
|
host_agent_apply,
|
|
HOST_ROOT_UID=uid,
|
|
HOST_ROOT_GID=gid,
|
|
HOST_RUNTIME_UID=runtime_uid,
|
|
HOST_RUNTIME_GID=runtime_gid,
|
|
HOST_ACTIVE_DIRECTORY=active,
|
|
HOST_ACTIVE_DIRECTORY_MODE=0o700,
|
|
HOST_ACTIVE_CONFIG_PATH=active_config_path,
|
|
HOST_ACTIVE_SECRETS_PATH=active_secrets_path,
|
|
HOST_CANDIDATE_DIRECTORY=candidates,
|
|
HOST_CANDIDATE_DIRECTORY_MODE=0o700,
|
|
HOST_CONFIG_CANDIDATE_PATH=candidate_config_path,
|
|
HOST_SECRETS_CANDIDATE_PATH=candidate_secrets_path,
|
|
HOST_APPLY_DIRECTORY=apply,
|
|
HOST_APPLY_LOCK_PATH=apply / 'apply.lock',
|
|
HOST_BACKUP_ROOT=backups,
|
|
HOST_BACKUP_DIRECTORY_MODE=0o700,
|
|
HOST_TEMPLATE_PATH=template_path,
|
|
PrivateFileLock=(
|
|
_REAL_PRIVATE_FILE_LOCK if real_lock else _TestLock
|
|
),
|
|
))
|
|
yield values
|
|
|
|
@staticmethod
|
|
def request(store, action='apply-both'):
|
|
enum = HostAgentAction(action)
|
|
return HostAgentRequest(
|
|
operation_id=str(uuid.uuid4()),
|
|
action=enum,
|
|
active_config_sha256=hashlib.sha256(
|
|
store['active_config'],
|
|
).hexdigest(),
|
|
active_secrets_sha256=hashlib.sha256(
|
|
store['active_secrets'],
|
|
).hexdigest(),
|
|
candidate_config_sha256=(
|
|
hashlib.sha256(store['proposed_config']).hexdigest()
|
|
if enum in (HostAgentAction.APPLY_CONFIG, HostAgentAction.APPLY_BOTH)
|
|
else None
|
|
),
|
|
candidate_secrets_sha256=(
|
|
hashlib.sha256(store['proposed_secrets']).hexdigest()
|
|
if enum in (HostAgentAction.APPLY_SECRETS, HostAgentAction.APPLY_BOTH)
|
|
else None
|
|
),
|
|
)
|
|
|
|
def setUp(self):
|
|
_TestLock.held.clear()
|
|
|
|
def test_claims_validates_backs_up_and_atomically_replaces_fixed_documents(self):
|
|
with self.store() as store:
|
|
database = _Database()
|
|
request = self.request(store)
|
|
|
|
with host_agent_apply.HostApplySession(request, database) as session:
|
|
session.backup()
|
|
result = session.replace(self.proof(request))
|
|
|
|
operation_backup = store['backups'] / request.operation_id
|
|
self.assertEqual(
|
|
(operation_backup / 'config.yaml').read_bytes(),
|
|
store['active_config'],
|
|
)
|
|
self.assertEqual(
|
|
(operation_backup / 'secrets.yaml').read_bytes(),
|
|
store['active_secrets'],
|
|
)
|
|
self.assertEqual(
|
|
store['active_config_path'].read_bytes(), store['proposed_config'],
|
|
)
|
|
self.assertEqual(
|
|
store['active_secrets_path'].read_bytes(), store['proposed_secrets'],
|
|
)
|
|
self.assertEqual(
|
|
store['candidate_config_path'].read_bytes(), store['proposed_config'],
|
|
)
|
|
self.assertEqual(
|
|
store['candidate_secrets_path'].read_bytes(), store['proposed_secrets'],
|
|
)
|
|
self.assertEqual(result, {
|
|
'active_config_sha256': request.candidate_config_sha256,
|
|
'active_secrets_sha256': request.candidate_secrets_sha256,
|
|
})
|
|
self.assertEqual(database.claims, [{
|
|
'operation_id': request.operation_id,
|
|
'action': 'apply-both',
|
|
'expected_identity': {
|
|
'active_config_sha256': request.active_config_sha256,
|
|
'active_secrets_sha256': request.active_secrets_sha256,
|
|
'candidate_config_sha256': request.candidate_config_sha256,
|
|
'candidate_secrets_sha256': request.candidate_secrets_sha256,
|
|
},
|
|
}])
|
|
|
|
def test_replace_requires_authentic_operation_bound_stopped_proof(self):
|
|
with self.store() as store:
|
|
request = self.request(store)
|
|
invalid = (
|
|
None,
|
|
object(),
|
|
host_agent_apply._new_stopped_runtime_proof(str(uuid.uuid4())),
|
|
host_agent_apply._new_stopped_runtime_proof(
|
|
request.operation_id, purpose='rollback',
|
|
),
|
|
host_agent_apply._StoppedRuntimeProof(
|
|
request.operation_id, 'forward', object(), object(),
|
|
),
|
|
)
|
|
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
|
session.backup()
|
|
for proof in invalid:
|
|
with self.subTest(proof=type(proof).__name__):
|
|
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
session.replace(proof)
|
|
self.assertEqual(raised.exception.category, 'state')
|
|
self.assertEqual(
|
|
store['active_config_path'].read_bytes(),
|
|
store['active_config'],
|
|
)
|
|
self.assertEqual(
|
|
store['active_secrets_path'].read_bytes(),
|
|
store['active_secrets'],
|
|
)
|
|
def test_restart_revalidates_without_creating_backup_or_replacing_files(self):
|
|
with self.store(candidate_config=False, candidate_secrets=False) as store:
|
|
database = _Database()
|
|
request = self.request(store, 'restart')
|
|
|
|
with host_agent_apply.HostApplySession(request, database) as session:
|
|
session.backup()
|
|
result = session.replace(self.proof(request))
|
|
|
|
self.assertFalse((store['backups'] / request.operation_id).exists())
|
|
self.assertEqual(store['active_config_path'].read_bytes(), store['active_config'])
|
|
self.assertEqual(store['active_secrets_path'].read_bytes(), store['active_secrets'])
|
|
self.assertEqual(result, {
|
|
'active_config_sha256': request.active_config_sha256,
|
|
'active_secrets_sha256': request.active_secrets_sha256,
|
|
})
|
|
|
|
def test_singleton_lock_rejects_second_session_before_database_claim(self):
|
|
with self.store() as store:
|
|
request = self.request(store)
|
|
first_database = _Database()
|
|
second_database = _Database()
|
|
first = host_agent_apply.HostApplySession(request, first_database)
|
|
first.__enter__()
|
|
try:
|
|
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
host_agent_apply.HostApplySession(
|
|
request, second_database,
|
|
).__enter__()
|
|
self.assertEqual(raised.exception.category, 'busy')
|
|
self.assertEqual(second_database.claims, [])
|
|
finally:
|
|
first.close()
|
|
|
|
def test_persisted_authority_failure_happens_before_document_reads(self):
|
|
with self.store() as store:
|
|
database = _Database()
|
|
database.failure = RuntimeError('database detail')
|
|
request = self.request(store)
|
|
with mock.patch.object(
|
|
host_agent_apply, '_snapshot_file',
|
|
) as snapshot, self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
with host_agent_apply.HostApplySession(request, database):
|
|
pass
|
|
self.assertEqual(raised.exception.category, 'authority')
|
|
snapshot.assert_not_called()
|
|
self.assertNotIn('database detail', str(raised.exception))
|
|
|
|
def test_non_postgres_authority_fails_before_document_reads(self):
|
|
with self.store() as store:
|
|
database = _Database()
|
|
database.conn.is_postgres = False
|
|
with mock.patch.object(
|
|
host_agent_apply, '_snapshot_file',
|
|
) as snapshot, self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
with host_agent_apply.HostApplySession(
|
|
self.request(store), database,
|
|
):
|
|
pass
|
|
self.assertEqual(raised.exception.category, 'authority')
|
|
self.assertEqual(database.claims, [])
|
|
snapshot.assert_not_called()
|
|
|
|
def test_same_operation_failed_hold_replay_skips_document_preparation(self):
|
|
with self.store() as store:
|
|
request = self.request(store)
|
|
database = _Database()
|
|
with mock.patch.object(
|
|
host_agent_apply, 'failed_hold_operation',
|
|
return_value=request.operation_id,
|
|
), mock.patch.object(
|
|
host_agent_apply.HostApplySession, '_prepare',
|
|
) as prepare:
|
|
with host_agent_apply.HostApplySession(
|
|
request, database,
|
|
) as session:
|
|
self.assertTrue(session._failed_hold_replay)
|
|
self.assertIsNone(session._active)
|
|
self.assertEqual(session.publication_state, 'original')
|
|
|
|
prepare.assert_not_called()
|
|
self.assertEqual(len(database.claims), 1)
|
|
|
|
def test_invalid_candidate_is_rejected_without_rendering_secret_bytes(self):
|
|
sentinel = b'never-render-this-secret-token'
|
|
with self.store() as store:
|
|
store['proposed_secrets'] = b'auth_pools: [' + sentinel
|
|
self.write_private(
|
|
store['candidate_secrets_path'], store['proposed_secrets'],
|
|
)
|
|
request = self.request(store, 'apply-secrets')
|
|
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
with host_agent_apply.HostApplySession(request, _Database()):
|
|
pass
|
|
self.assertEqual(raised.exception.category, 'validation')
|
|
self.assertNotIn(sentinel.decode('ascii'), str(raised.exception))
|
|
self.assertNotIn(sentinel.decode('ascii'), repr(raised.exception))
|
|
|
|
def test_hash_drift_before_publication_preserves_active_files(self):
|
|
with self.store() as store:
|
|
request = self.request(store)
|
|
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
|
session.backup()
|
|
store['candidate_config_path'].write_bytes(b'changed concurrently')
|
|
os.chmod(store['candidate_config_path'], 0o600)
|
|
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
session.replace(self.proof(request))
|
|
self.assertEqual(raised.exception.category, 'identity')
|
|
self.assertEqual(store['active_config_path'].read_bytes(), store['active_config'])
|
|
self.assertEqual(store['active_secrets_path'].read_bytes(), store['active_secrets'])
|
|
self.assertEqual(list(store['active'].glob('*.stage')), [])
|
|
|
|
def test_completed_publication_replay_verifies_backup_without_replacing_again(self):
|
|
with self.store() as store:
|
|
request = self.request(store)
|
|
database = _Database()
|
|
with host_agent_apply.HostApplySession(request, database) as session:
|
|
session.backup()
|
|
expected = session.replace(self.proof(request))
|
|
|
|
database.replayed = True
|
|
with mock.patch.object(
|
|
host_agent_apply, 'durable_replace',
|
|
) as replace, host_agent_apply.HostApplySession(
|
|
request, database,
|
|
) as replay:
|
|
replay.backup()
|
|
actual = replay.replace(self.proof(request))
|
|
|
|
self.assertEqual(actual, expected)
|
|
replace.assert_not_called()
|
|
self.assertEqual(
|
|
store['active_config_path'].read_bytes(), store['proposed_config'],
|
|
)
|
|
self.assertEqual(
|
|
store['active_secrets_path'].read_bytes(), store['proposed_secrets'],
|
|
)
|
|
|
|
def test_interrupted_second_backup_replays_without_changing_active_files(self):
|
|
with self.store() as store:
|
|
request = self.request(store)
|
|
database = _Database()
|
|
create = host_agent_apply._create_owned_file
|
|
|
|
def interrupt_second(destination, *args, **kwargs):
|
|
if Path(destination).name == 'secrets.yaml':
|
|
raise SystemExit()
|
|
return create(destination, *args, **kwargs)
|
|
|
|
with host_agent_apply.HostApplySession(request, database) as session:
|
|
with mock.patch.object(
|
|
host_agent_apply, '_create_owned_file',
|
|
side_effect=interrupt_second,
|
|
), self.assertRaises(SystemExit):
|
|
session.backup()
|
|
|
|
operation_backup = store['backups'] / request.operation_id
|
|
self.assertEqual(
|
|
(operation_backup / 'config.yaml').read_bytes(),
|
|
store['active_config'],
|
|
)
|
|
self.assertFalse((operation_backup / 'secrets.yaml').exists())
|
|
|
|
database.replayed = True
|
|
with host_agent_apply.HostApplySession(request, database) as replay:
|
|
replay.backup()
|
|
|
|
self.assertEqual(
|
|
(operation_backup / 'config.yaml').read_bytes(),
|
|
store['active_config'],
|
|
)
|
|
self.assertEqual(
|
|
(operation_backup / 'secrets.yaml').read_bytes(),
|
|
store['active_secrets'],
|
|
)
|
|
self.assertEqual(
|
|
store['active_config_path'].read_bytes(), store['active_config'],
|
|
)
|
|
self.assertEqual(
|
|
store['active_secrets_path'].read_bytes(), store['active_secrets'],
|
|
)
|
|
|
|
def test_partial_publication_replay_is_recovery_only_and_restores_both(self):
|
|
with self.store() as store:
|
|
request = self.request(store)
|
|
database = _Database()
|
|
with host_agent_apply.HostApplySession(request, database) as session:
|
|
session.backup()
|
|
database.replayed = True
|
|
self.write_private(
|
|
store['active_config_path'], store['proposed_config'],
|
|
)
|
|
|
|
with host_agent_apply.HostApplySession(request, database) as replay:
|
|
self.assertEqual(replay.publication_state, 'partial')
|
|
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
replay.replace(self.proof(request))
|
|
self.assertEqual(raised.exception.category, 'partial')
|
|
result = replay.restore_backups(self.rollback_proof(request))
|
|
|
|
self.assertEqual(result, {
|
|
'active_config_sha256': request.active_config_sha256,
|
|
'active_secrets_sha256': request.active_secrets_sha256,
|
|
})
|
|
self.assertEqual(
|
|
store['active_config_path'].read_bytes(), store['active_config'],
|
|
)
|
|
self.assertEqual(
|
|
store['active_secrets_path'].read_bytes(), store['active_secrets'],
|
|
)
|
|
|
|
def test_completed_publication_can_be_restored_byte_identically_once(self):
|
|
with self.store() as store:
|
|
request = self.request(store)
|
|
with host_agent_apply.HostApplySession(
|
|
request, _Database(),
|
|
) as session:
|
|
session.backup()
|
|
session.replace(self.proof(request))
|
|
proof = self.rollback_proof(request)
|
|
result = session.restore_backups(proof)
|
|
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
session.restore_backups(proof)
|
|
|
|
self.assertEqual(raised.exception.category, 'state')
|
|
self.assertEqual(result, {
|
|
'active_config_sha256': request.active_config_sha256,
|
|
'active_secrets_sha256': request.active_secrets_sha256,
|
|
})
|
|
self.assertEqual(
|
|
store['active_config_path'].read_bytes(), store['active_config'],
|
|
)
|
|
self.assertEqual(
|
|
store['active_secrets_path'].read_bytes(), store['active_secrets'],
|
|
)
|
|
operation_backup = store['backups'] / request.operation_id
|
|
self.assertEqual(
|
|
(operation_backup / 'config.yaml').read_bytes(),
|
|
store['active_config'],
|
|
)
|
|
self.assertEqual(
|
|
(operation_backup / 'secrets.yaml').read_bytes(),
|
|
store['active_secrets'],
|
|
)
|
|
|
|
def test_restore_failure_after_first_publish_records_partial_state(self):
|
|
with self.store() as store:
|
|
request = self.request(store)
|
|
with host_agent_apply.HostApplySession(
|
|
request, _Database(),
|
|
) as session:
|
|
session.backup()
|
|
session.replace(self.proof(request))
|
|
replace = host_agent_apply.durable_replace
|
|
rollback_replaces = 0
|
|
|
|
def fail_second_rollback(source, destination):
|
|
nonlocal rollback_replaces
|
|
if source.name.endswith('.rollback'):
|
|
rollback_replaces += 1
|
|
if rollback_replaces == 2:
|
|
raise OSError('rollback detail')
|
|
return replace(source, destination)
|
|
|
|
with mock.patch.object(
|
|
host_agent_apply, 'durable_replace',
|
|
side_effect=fail_second_rollback,
|
|
):
|
|
with self.assertRaises(host_agent_apply.HostApplyError):
|
|
session.restore_backups(self.rollback_proof(request))
|
|
|
|
self.assertEqual(session.publication_state, 'partial')
|
|
|
|
self.assertEqual(
|
|
store['active_config_path'].read_bytes(), store['active_config'],
|
|
)
|
|
self.assertEqual(
|
|
store['active_secrets_path'].read_bytes(),
|
|
store['proposed_secrets'],
|
|
)
|
|
operation_backup = store['backups'] / request.operation_id
|
|
self.assertEqual(
|
|
(operation_backup / 'config.yaml').read_bytes(),
|
|
store['active_config'],
|
|
)
|
|
self.assertEqual(
|
|
(operation_backup / 'secrets.yaml').read_bytes(),
|
|
store['active_secrets'],
|
|
)
|
|
self.assertEqual(
|
|
store['candidate_config_path'].read_bytes(),
|
|
store['proposed_config'],
|
|
)
|
|
self.assertEqual(
|
|
store['candidate_secrets_path'].read_bytes(),
|
|
store['proposed_secrets'],
|
|
)
|
|
|
|
def test_restore_refuses_unknown_active_bytes_without_overwrite(self):
|
|
with self.store() as store:
|
|
request = self.request(store, 'apply-config')
|
|
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
|
session.backup()
|
|
session.replace(self.proof(request))
|
|
foreign = b'foreign active bytes'
|
|
self.write_private(store['active_config_path'], foreign)
|
|
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
session.restore_backups(self.rollback_proof(request))
|
|
|
|
self.assertEqual(raised.exception.category, 'identity')
|
|
self.assertEqual(store['active_config_path'].read_bytes(), foreign)
|
|
|
|
@unittest.skipUnless(
|
|
os.name != 'nt' and getattr(os, 'geteuid', lambda: -1)() == 0,
|
|
'requires distinct root and runtime ownership',
|
|
)
|
|
def test_replay_adopts_root_owned_original_left_during_rollback(self):
|
|
with self.store() as store:
|
|
request = self.request(store, 'apply-config')
|
|
database = _Database()
|
|
with host_agent_apply.HostApplySession(request, database) as session:
|
|
session.backup()
|
|
session.replace(self.proof(request))
|
|
stage = store['active'] / (
|
|
f'.config.yaml.{request.operation_id}.rollback'
|
|
)
|
|
host_agent_apply._create_owned_file(
|
|
stage,
|
|
store['active_config'],
|
|
uid=host_agent_apply.HOST_ROOT_UID,
|
|
gid=host_agent_apply.HOST_ROOT_GID,
|
|
mode=0o600,
|
|
expected_sha256=request.active_config_sha256,
|
|
)
|
|
host_agent_apply.durable_replace(
|
|
stage, store['active_config_path'],
|
|
)
|
|
|
|
self.assertEqual(
|
|
os.stat(store['active_config_path']).st_uid,
|
|
host_agent_apply.HOST_ROOT_UID,
|
|
)
|
|
database.replayed = True
|
|
with host_agent_apply.HostApplySession(request, database) as replay:
|
|
result = replay.restore_backups(self.rollback_proof(request))
|
|
|
|
self.assertEqual(result, {
|
|
'active_config_sha256': request.active_config_sha256,
|
|
'active_secrets_sha256': request.active_secrets_sha256,
|
|
})
|
|
self.assertEqual(
|
|
os.stat(store['active_config_path']).st_uid,
|
|
host_agent_apply.HOST_RUNTIME_UID,
|
|
)
|
|
self.assertEqual(
|
|
store['active_config_path'].read_bytes(), store['active_config'],
|
|
)
|
|
|
|
@unittest.skipUnless(
|
|
os.name != 'nt' and getattr(os, 'geteuid', lambda: -1)() == 0,
|
|
'requires distinct root and runtime ownership',
|
|
)
|
|
def test_replay_adopts_root_owned_candidate_left_after_publication(self):
|
|
with self.store() as store:
|
|
request = self.request(store, 'apply-config')
|
|
database = _Database()
|
|
with host_agent_apply.HostApplySession(request, database) as session:
|
|
session.backup()
|
|
stage = store['active'] / (
|
|
f'.config.yaml.{request.operation_id}.stage'
|
|
)
|
|
host_agent_apply._create_owned_file(
|
|
stage,
|
|
store['proposed_config'],
|
|
uid=host_agent_apply.HOST_ROOT_UID,
|
|
gid=host_agent_apply.HOST_ROOT_GID,
|
|
mode=0o600,
|
|
expected_sha256=request.candidate_config_sha256,
|
|
)
|
|
host_agent_apply.durable_replace(
|
|
stage, store['active_config_path'],
|
|
)
|
|
|
|
self.assertEqual(
|
|
os.stat(store['active_config_path']).st_uid,
|
|
host_agent_apply.HOST_ROOT_UID,
|
|
)
|
|
database.replayed = True
|
|
with host_agent_apply.HostApplySession(request, database) as replay:
|
|
replay.backup()
|
|
result = replay.replace(self.proof(request))
|
|
|
|
self.assertEqual(
|
|
os.stat(store['active_config_path']).st_uid,
|
|
host_agent_apply.HOST_RUNTIME_UID,
|
|
)
|
|
self.assertEqual(
|
|
result['active_config_sha256'], request.candidate_config_sha256,
|
|
)
|
|
|
|
def test_stale_fixed_stage_is_removed_before_replacement(self):
|
|
with self.store() as store:
|
|
request = self.request(store, 'apply-config')
|
|
stage = store['active'] / (
|
|
f'.config.yaml.{request.operation_id}.stage'
|
|
)
|
|
self.write_private(stage, b'incomplete prior stage')
|
|
|
|
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
|
session.backup()
|
|
session.replace(self.proof(request))
|
|
|
|
self.assertFalse(stage.exists())
|
|
self.assertEqual(
|
|
store['active_config_path'].read_bytes(), store['proposed_config'],
|
|
)
|
|
|
|
def test_stage_remains_root_owned_until_publication(self):
|
|
with self.store() as store:
|
|
request = self.request(store, 'apply-config')
|
|
real_create = host_agent_apply._create_owned_file
|
|
calls = []
|
|
|
|
def record(path, payload, **metadata):
|
|
calls.append((Path(path), dict(metadata)))
|
|
return real_create(path, payload, **metadata)
|
|
|
|
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
|
session.backup()
|
|
with mock.patch.object(
|
|
host_agent_apply, '_create_owned_file', side_effect=record,
|
|
):
|
|
session.replace(self.proof(request))
|
|
|
|
stage_calls = [item for item in calls if item[0].suffix == '.stage']
|
|
self.assertEqual(len(stage_calls), 1)
|
|
self.assertEqual(stage_calls[0][1]['uid'], host_agent_apply.HOST_ROOT_UID)
|
|
self.assertEqual(stage_calls[0][1]['gid'], host_agent_apply.HOST_ROOT_GID)
|
|
|
|
@unittest.skipIf(os.name == 'nt', 'POSIX displaced-inode guard')
|
|
def test_active_write_after_revalidation_is_detected_as_partial(self):
|
|
with self.store() as store:
|
|
request = self.request(store, 'apply-config')
|
|
real_replace = host_agent_apply.durable_replace
|
|
|
|
def mutate_then_replace(source, destination):
|
|
self.write_private(Path(destination), b'late concurrent write')
|
|
real_replace(source, destination)
|
|
|
|
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
|
session.backup()
|
|
with mock.patch.object(
|
|
host_agent_apply,
|
|
'durable_replace',
|
|
side_effect=mutate_then_replace,
|
|
), self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
session.replace(self.proof(request))
|
|
|
|
self.assertEqual(raised.exception.category, 'partial')
|
|
self.assertEqual(
|
|
store['active_config_path'].read_bytes(), store['proposed_config'],
|
|
)
|
|
self.assertEqual(
|
|
(
|
|
store['backups'] / request.operation_id / 'config.yaml'
|
|
).read_bytes(),
|
|
store['active_config'],
|
|
)
|
|
|
|
def test_existing_backup_must_be_byte_identical(self):
|
|
with self.store() as store:
|
|
request = self.request(store, 'apply-config')
|
|
operation_backup = store['backups'] / request.operation_id
|
|
operation_backup.mkdir(mode=0o700)
|
|
os.chmod(operation_backup, 0o700)
|
|
self.write_private(operation_backup / 'config.yaml', b'foreign backup')
|
|
|
|
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
|
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
session.backup()
|
|
self.assertEqual(raised.exception.category, 'identity')
|
|
self.assertEqual(
|
|
(operation_backup / 'config.yaml').read_bytes(), b'foreign backup',
|
|
)
|
|
|
|
def test_combined_second_publication_failure_is_reported_as_partial(self):
|
|
with self.store() as store:
|
|
request = self.request(store)
|
|
real_replace = host_agent_apply.durable_replace
|
|
calls = []
|
|
|
|
def fail_second(source, destination):
|
|
calls.append(Path(destination).name)
|
|
if len(calls) == 2:
|
|
raise OSError('second publication failed')
|
|
real_replace(source, destination)
|
|
|
|
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
|
session.backup()
|
|
with mock.patch.object(
|
|
host_agent_apply, 'durable_replace', side_effect=fail_second,
|
|
), self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
session.replace(self.proof(request))
|
|
|
|
self.assertEqual(raised.exception.category, 'partial')
|
|
self.assertEqual(store['active_config_path'].read_bytes(), store['proposed_config'])
|
|
self.assertEqual(store['active_secrets_path'].read_bytes(), store['active_secrets'])
|
|
operation_backup = store['backups'] / request.operation_id
|
|
self.assertEqual(
|
|
(operation_backup / 'config.yaml').read_bytes(), store['active_config'],
|
|
)
|
|
self.assertEqual(
|
|
(operation_backup / 'secrets.yaml').read_bytes(), store['active_secrets'],
|
|
)
|
|
|
|
@unittest.skipIf(os.name == 'nt', 'POSIX link metadata test')
|
|
def test_candidate_hardlink_is_rejected_without_claimed_content_publication(self):
|
|
with self.store(candidate_config=False) as store:
|
|
os.link(store['active_config_path'], store['candidate_config_path'])
|
|
request = self.request(store, 'apply-config')
|
|
request = HostAgentRequest(
|
|
operation_id=request.operation_id,
|
|
action=request.action,
|
|
active_config_sha256=request.active_config_sha256,
|
|
active_secrets_sha256=request.active_secrets_sha256,
|
|
candidate_config_sha256=request.active_config_sha256,
|
|
candidate_secrets_sha256=None,
|
|
)
|
|
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
with host_agent_apply.HostApplySession(request, _Database()):
|
|
pass
|
|
self.assertEqual(raised.exception.category, 'filesystem')
|
|
|
|
@unittest.skipIf(os.name == 'nt', 'POSIX flock integration test')
|
|
def test_real_filesystem_lock_excludes_concurrent_host_session(self):
|
|
with self.store(real_lock=True) as store:
|
|
request = self.request(store, 'restart')
|
|
first = host_agent_apply.HostApplySession(request, _Database())
|
|
first.__enter__()
|
|
try:
|
|
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
|
host_agent_apply.HostApplySession(
|
|
request, _Database(),
|
|
).__enter__()
|
|
self.assertEqual(raised.exception.category, 'busy')
|
|
finally:
|
|
first.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|