Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,849 @@
|
||||
import copy
|
||||
from contextlib import contextmanager, ExitStack
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import uuid
|
||||
from unittest import mock
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
APP_DIR = ROOT / 'app'
|
||||
sys.path.insert(0, str(APP_DIR))
|
||||
|
||||
import host_agent_apply
|
||||
from host_agent_protocol import HostAgentAction, HostAgentRequest
|
||||
import runtime_document
|
||||
|
||||
|
||||
_REAL_PRIVATE_FILE_LOCK = host_agent_apply.PrivateFileLock
|
||||
|
||||
|
||||
class _Connection:
|
||||
is_postgres = True
|
||||
|
||||
|
||||
class _Database:
|
||||
def __init__(self):
|
||||
self.conn = _Connection()
|
||||
self.claims = []
|
||||
self.failure = None
|
||||
self.replayed = False
|
||||
|
||||
def claim_runtime_operation_execution(self, **values):
|
||||
self.claims.append(values)
|
||||
if self.failure is not None:
|
||||
raise self.failure
|
||||
return {'status': 'running', 'replayed': self.replayed}
|
||||
|
||||
|
||||
class _TestLock:
|
||||
held = set()
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = os.fspath(path)
|
||||
self.acquired = False
|
||||
|
||||
def acquire(self):
|
||||
if self.path in self.held:
|
||||
raise BlockingIOError('held')
|
||||
self.held.add(self.path)
|
||||
self.acquired = True
|
||||
return self
|
||||
|
||||
def release(self):
|
||||
if self.acquired:
|
||||
self.held.remove(self.path)
|
||||
self.acquired = False
|
||||
|
||||
|
||||
class HostAgentApplyTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def proof(request):
|
||||
return host_agent_apply._new_stopped_runtime_proof(request.operation_id)
|
||||
|
||||
@staticmethod
|
||||
def rollback_proof(request):
|
||||
return host_agent_apply._new_stopped_runtime_proof(
|
||||
request.operation_id, purpose='rollback',
|
||||
)
|
||||
|
||||
def documents(self):
|
||||
template_payload = (APP_DIR / 'config.linux.yaml').read_bytes()
|
||||
template = runtime_document.load_yaml_document(
|
||||
template_payload,
|
||||
max_bytes=runtime_document.MAX_CONFIG_DOCUMENT_BYTES,
|
||||
)
|
||||
config = copy.deepcopy(template)
|
||||
docker_pool = config['sources']['dockerhub']['auth_pool']
|
||||
pools = {
|
||||
source['auth_pool']
|
||||
for source in config['sources'].values()
|
||||
if source.get('auth_pool')
|
||||
}
|
||||
secrets = {'auth_pools': {
|
||||
pool: [{
|
||||
'name': pool + '_1',
|
||||
**({'username': 'fixture-user'} if pool == docker_pool else {}),
|
||||
'token': 'fixture-token',
|
||||
}]
|
||||
for pool in pools
|
||||
}}
|
||||
return template_payload, config, secrets
|
||||
|
||||
@staticmethod
|
||||
def write_private(path, payload):
|
||||
path.write_bytes(payload)
|
||||
os.chmod(path, 0o600)
|
||||
|
||||
@contextmanager
|
||||
def store(
|
||||
self, *, candidate_config=True, candidate_secrets=True, real_lock=False,
|
||||
):
|
||||
template_payload, config, secrets = self.documents()
|
||||
active_config = yaml.safe_dump(config).encode('utf-8')
|
||||
active_secrets = yaml.safe_dump(secrets).encode('utf-8')
|
||||
proposed_config = active_config + b'# candidate config\n'
|
||||
proposed_secrets = active_secrets + b'# candidate secrets\n'
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
active = root / 'active'
|
||||
candidates = root / 'candidates'
|
||||
apply = root / 'apply'
|
||||
backups = root / 'backups'
|
||||
for directory in (active, candidates, apply, backups):
|
||||
directory.mkdir(mode=0o700)
|
||||
os.chmod(directory, 0o700)
|
||||
active_config_path = active / 'config.yaml'
|
||||
active_secrets_path = active / 'secrets.yaml'
|
||||
candidate_config_path = candidates / 'config.yaml'
|
||||
candidate_secrets_path = candidates / 'secrets.yaml'
|
||||
template_path = root / 'template.yaml'
|
||||
self.write_private(active_config_path, active_config)
|
||||
self.write_private(active_secrets_path, active_secrets)
|
||||
self.write_private(template_path, template_payload)
|
||||
if candidate_config:
|
||||
self.write_private(candidate_config_path, proposed_config)
|
||||
if candidate_secrets:
|
||||
self.write_private(candidate_secrets_path, proposed_secrets)
|
||||
details = os.stat(root)
|
||||
uid = getattr(details, 'st_uid', 0)
|
||||
gid = getattr(details, 'st_gid', 0)
|
||||
runtime_uid = (
|
||||
10001
|
||||
if os.name != 'nt' and os.geteuid() == 0
|
||||
else uid
|
||||
)
|
||||
runtime_gid = (
|
||||
10001
|
||||
if os.name != 'nt' and os.geteuid() == 0
|
||||
else gid
|
||||
)
|
||||
if os.name != 'nt' and runtime_uid != uid:
|
||||
os.chown(candidates, runtime_uid, runtime_gid)
|
||||
for path in (
|
||||
active_config_path,
|
||||
active_secrets_path,
|
||||
candidate_config_path,
|
||||
candidate_secrets_path,
|
||||
):
|
||||
if path.exists():
|
||||
os.chown(path, runtime_uid, runtime_gid)
|
||||
values = {
|
||||
'root': root,
|
||||
'active': active,
|
||||
'candidates': candidates,
|
||||
'backups': backups,
|
||||
'active_config_path': active_config_path,
|
||||
'active_secrets_path': active_secrets_path,
|
||||
'candidate_config_path': candidate_config_path,
|
||||
'candidate_secrets_path': candidate_secrets_path,
|
||||
'active_config': active_config,
|
||||
'active_secrets': active_secrets,
|
||||
'proposed_config': proposed_config,
|
||||
'proposed_secrets': proposed_secrets,
|
||||
}
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(mock.patch.multiple(
|
||||
host_agent_apply,
|
||||
HOST_ROOT_UID=uid,
|
||||
HOST_ROOT_GID=gid,
|
||||
HOST_RUNTIME_UID=runtime_uid,
|
||||
HOST_RUNTIME_GID=runtime_gid,
|
||||
HOST_ACTIVE_DIRECTORY=active,
|
||||
HOST_ACTIVE_DIRECTORY_MODE=0o700,
|
||||
HOST_ACTIVE_CONFIG_PATH=active_config_path,
|
||||
HOST_ACTIVE_SECRETS_PATH=active_secrets_path,
|
||||
HOST_CANDIDATE_DIRECTORY=candidates,
|
||||
HOST_CANDIDATE_DIRECTORY_MODE=0o700,
|
||||
HOST_CONFIG_CANDIDATE_PATH=candidate_config_path,
|
||||
HOST_SECRETS_CANDIDATE_PATH=candidate_secrets_path,
|
||||
HOST_APPLY_DIRECTORY=apply,
|
||||
HOST_APPLY_LOCK_PATH=apply / 'apply.lock',
|
||||
HOST_BACKUP_ROOT=backups,
|
||||
HOST_BACKUP_DIRECTORY_MODE=0o700,
|
||||
HOST_TEMPLATE_PATH=template_path,
|
||||
PrivateFileLock=(
|
||||
_REAL_PRIVATE_FILE_LOCK if real_lock else _TestLock
|
||||
),
|
||||
))
|
||||
yield values
|
||||
|
||||
@staticmethod
|
||||
def request(store, action='apply-both'):
|
||||
enum = HostAgentAction(action)
|
||||
return HostAgentRequest(
|
||||
operation_id=str(uuid.uuid4()),
|
||||
action=enum,
|
||||
active_config_sha256=hashlib.sha256(
|
||||
store['active_config'],
|
||||
).hexdigest(),
|
||||
active_secrets_sha256=hashlib.sha256(
|
||||
store['active_secrets'],
|
||||
).hexdigest(),
|
||||
candidate_config_sha256=(
|
||||
hashlib.sha256(store['proposed_config']).hexdigest()
|
||||
if enum in (HostAgentAction.APPLY_CONFIG, HostAgentAction.APPLY_BOTH)
|
||||
else None
|
||||
),
|
||||
candidate_secrets_sha256=(
|
||||
hashlib.sha256(store['proposed_secrets']).hexdigest()
|
||||
if enum in (HostAgentAction.APPLY_SECRETS, HostAgentAction.APPLY_BOTH)
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def setUp(self):
|
||||
_TestLock.held.clear()
|
||||
|
||||
def test_claims_validates_backs_up_and_atomically_replaces_fixed_documents(self):
|
||||
with self.store() as store:
|
||||
database = _Database()
|
||||
request = self.request(store)
|
||||
|
||||
with host_agent_apply.HostApplySession(request, database) as session:
|
||||
session.backup()
|
||||
result = session.replace(self.proof(request))
|
||||
|
||||
operation_backup = store['backups'] / request.operation_id
|
||||
self.assertEqual(
|
||||
(operation_backup / 'config.yaml').read_bytes(),
|
||||
store['active_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
(operation_backup / 'secrets.yaml').read_bytes(),
|
||||
store['active_secrets'],
|
||||
)
|
||||
self.assertEqual(
|
||||
store['active_config_path'].read_bytes(), store['proposed_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
store['active_secrets_path'].read_bytes(), store['proposed_secrets'],
|
||||
)
|
||||
self.assertEqual(
|
||||
store['candidate_config_path'].read_bytes(), store['proposed_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
store['candidate_secrets_path'].read_bytes(), store['proposed_secrets'],
|
||||
)
|
||||
self.assertEqual(result, {
|
||||
'active_config_sha256': request.candidate_config_sha256,
|
||||
'active_secrets_sha256': request.candidate_secrets_sha256,
|
||||
})
|
||||
self.assertEqual(database.claims, [{
|
||||
'operation_id': request.operation_id,
|
||||
'action': 'apply-both',
|
||||
'expected_identity': {
|
||||
'active_config_sha256': request.active_config_sha256,
|
||||
'active_secrets_sha256': request.active_secrets_sha256,
|
||||
'candidate_config_sha256': request.candidate_config_sha256,
|
||||
'candidate_secrets_sha256': request.candidate_secrets_sha256,
|
||||
},
|
||||
}])
|
||||
|
||||
def test_replace_requires_authentic_operation_bound_stopped_proof(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store)
|
||||
invalid = (
|
||||
None,
|
||||
object(),
|
||||
host_agent_apply._new_stopped_runtime_proof(str(uuid.uuid4())),
|
||||
host_agent_apply._new_stopped_runtime_proof(
|
||||
request.operation_id, purpose='rollback',
|
||||
),
|
||||
host_agent_apply._StoppedRuntimeProof(
|
||||
request.operation_id, 'forward', object(), object(),
|
||||
),
|
||||
)
|
||||
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
||||
session.backup()
|
||||
for proof in invalid:
|
||||
with self.subTest(proof=type(proof).__name__):
|
||||
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
session.replace(proof)
|
||||
self.assertEqual(raised.exception.category, 'state')
|
||||
self.assertEqual(
|
||||
store['active_config_path'].read_bytes(),
|
||||
store['active_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
store['active_secrets_path'].read_bytes(),
|
||||
store['active_secrets'],
|
||||
)
|
||||
def test_restart_revalidates_without_creating_backup_or_replacing_files(self):
|
||||
with self.store(candidate_config=False, candidate_secrets=False) as store:
|
||||
database = _Database()
|
||||
request = self.request(store, 'restart')
|
||||
|
||||
with host_agent_apply.HostApplySession(request, database) as session:
|
||||
session.backup()
|
||||
result = session.replace(self.proof(request))
|
||||
|
||||
self.assertFalse((store['backups'] / request.operation_id).exists())
|
||||
self.assertEqual(store['active_config_path'].read_bytes(), store['active_config'])
|
||||
self.assertEqual(store['active_secrets_path'].read_bytes(), store['active_secrets'])
|
||||
self.assertEqual(result, {
|
||||
'active_config_sha256': request.active_config_sha256,
|
||||
'active_secrets_sha256': request.active_secrets_sha256,
|
||||
})
|
||||
|
||||
def test_singleton_lock_rejects_second_session_before_database_claim(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store)
|
||||
first_database = _Database()
|
||||
second_database = _Database()
|
||||
first = host_agent_apply.HostApplySession(request, first_database)
|
||||
first.__enter__()
|
||||
try:
|
||||
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
host_agent_apply.HostApplySession(
|
||||
request, second_database,
|
||||
).__enter__()
|
||||
self.assertEqual(raised.exception.category, 'busy')
|
||||
self.assertEqual(second_database.claims, [])
|
||||
finally:
|
||||
first.close()
|
||||
|
||||
def test_persisted_authority_failure_happens_before_document_reads(self):
|
||||
with self.store() as store:
|
||||
database = _Database()
|
||||
database.failure = RuntimeError('database detail')
|
||||
request = self.request(store)
|
||||
with mock.patch.object(
|
||||
host_agent_apply, '_snapshot_file',
|
||||
) as snapshot, self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
with host_agent_apply.HostApplySession(request, database):
|
||||
pass
|
||||
self.assertEqual(raised.exception.category, 'authority')
|
||||
snapshot.assert_not_called()
|
||||
self.assertNotIn('database detail', str(raised.exception))
|
||||
|
||||
def test_non_postgres_authority_fails_before_document_reads(self):
|
||||
with self.store() as store:
|
||||
database = _Database()
|
||||
database.conn.is_postgres = False
|
||||
with mock.patch.object(
|
||||
host_agent_apply, '_snapshot_file',
|
||||
) as snapshot, self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
with host_agent_apply.HostApplySession(
|
||||
self.request(store), database,
|
||||
):
|
||||
pass
|
||||
self.assertEqual(raised.exception.category, 'authority')
|
||||
self.assertEqual(database.claims, [])
|
||||
snapshot.assert_not_called()
|
||||
|
||||
def test_same_operation_failed_hold_replay_skips_document_preparation(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store)
|
||||
database = _Database()
|
||||
with mock.patch.object(
|
||||
host_agent_apply, 'failed_hold_operation',
|
||||
return_value=request.operation_id,
|
||||
), mock.patch.object(
|
||||
host_agent_apply.HostApplySession, '_prepare',
|
||||
) as prepare:
|
||||
with host_agent_apply.HostApplySession(
|
||||
request, database,
|
||||
) as session:
|
||||
self.assertTrue(session._failed_hold_replay)
|
||||
self.assertIsNone(session._active)
|
||||
self.assertEqual(session.publication_state, 'original')
|
||||
|
||||
prepare.assert_not_called()
|
||||
self.assertEqual(len(database.claims), 1)
|
||||
|
||||
def test_invalid_candidate_is_rejected_without_rendering_secret_bytes(self):
|
||||
sentinel = b'never-render-this-secret-token'
|
||||
with self.store() as store:
|
||||
store['proposed_secrets'] = b'auth_pools: [' + sentinel
|
||||
self.write_private(
|
||||
store['candidate_secrets_path'], store['proposed_secrets'],
|
||||
)
|
||||
request = self.request(store, 'apply-secrets')
|
||||
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
with host_agent_apply.HostApplySession(request, _Database()):
|
||||
pass
|
||||
self.assertEqual(raised.exception.category, 'validation')
|
||||
self.assertNotIn(sentinel.decode('ascii'), str(raised.exception))
|
||||
self.assertNotIn(sentinel.decode('ascii'), repr(raised.exception))
|
||||
|
||||
def test_hash_drift_before_publication_preserves_active_files(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store)
|
||||
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
||||
session.backup()
|
||||
store['candidate_config_path'].write_bytes(b'changed concurrently')
|
||||
os.chmod(store['candidate_config_path'], 0o600)
|
||||
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
session.replace(self.proof(request))
|
||||
self.assertEqual(raised.exception.category, 'identity')
|
||||
self.assertEqual(store['active_config_path'].read_bytes(), store['active_config'])
|
||||
self.assertEqual(store['active_secrets_path'].read_bytes(), store['active_secrets'])
|
||||
self.assertEqual(list(store['active'].glob('*.stage')), [])
|
||||
|
||||
def test_completed_publication_replay_verifies_backup_without_replacing_again(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store)
|
||||
database = _Database()
|
||||
with host_agent_apply.HostApplySession(request, database) as session:
|
||||
session.backup()
|
||||
expected = session.replace(self.proof(request))
|
||||
|
||||
database.replayed = True
|
||||
with mock.patch.object(
|
||||
host_agent_apply, 'durable_replace',
|
||||
) as replace, host_agent_apply.HostApplySession(
|
||||
request, database,
|
||||
) as replay:
|
||||
replay.backup()
|
||||
actual = replay.replace(self.proof(request))
|
||||
|
||||
self.assertEqual(actual, expected)
|
||||
replace.assert_not_called()
|
||||
self.assertEqual(
|
||||
store['active_config_path'].read_bytes(), store['proposed_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
store['active_secrets_path'].read_bytes(), store['proposed_secrets'],
|
||||
)
|
||||
|
||||
def test_interrupted_second_backup_replays_without_changing_active_files(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store)
|
||||
database = _Database()
|
||||
create = host_agent_apply._create_owned_file
|
||||
|
||||
def interrupt_second(destination, *args, **kwargs):
|
||||
if Path(destination).name == 'secrets.yaml':
|
||||
raise SystemExit()
|
||||
return create(destination, *args, **kwargs)
|
||||
|
||||
with host_agent_apply.HostApplySession(request, database) as session:
|
||||
with mock.patch.object(
|
||||
host_agent_apply, '_create_owned_file',
|
||||
side_effect=interrupt_second,
|
||||
), self.assertRaises(SystemExit):
|
||||
session.backup()
|
||||
|
||||
operation_backup = store['backups'] / request.operation_id
|
||||
self.assertEqual(
|
||||
(operation_backup / 'config.yaml').read_bytes(),
|
||||
store['active_config'],
|
||||
)
|
||||
self.assertFalse((operation_backup / 'secrets.yaml').exists())
|
||||
|
||||
database.replayed = True
|
||||
with host_agent_apply.HostApplySession(request, database) as replay:
|
||||
replay.backup()
|
||||
|
||||
self.assertEqual(
|
||||
(operation_backup / 'config.yaml').read_bytes(),
|
||||
store['active_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
(operation_backup / 'secrets.yaml').read_bytes(),
|
||||
store['active_secrets'],
|
||||
)
|
||||
self.assertEqual(
|
||||
store['active_config_path'].read_bytes(), store['active_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
store['active_secrets_path'].read_bytes(), store['active_secrets'],
|
||||
)
|
||||
|
||||
def test_partial_publication_replay_is_recovery_only_and_restores_both(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store)
|
||||
database = _Database()
|
||||
with host_agent_apply.HostApplySession(request, database) as session:
|
||||
session.backup()
|
||||
database.replayed = True
|
||||
self.write_private(
|
||||
store['active_config_path'], store['proposed_config'],
|
||||
)
|
||||
|
||||
with host_agent_apply.HostApplySession(request, database) as replay:
|
||||
self.assertEqual(replay.publication_state, 'partial')
|
||||
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
replay.replace(self.proof(request))
|
||||
self.assertEqual(raised.exception.category, 'partial')
|
||||
result = replay.restore_backups(self.rollback_proof(request))
|
||||
|
||||
self.assertEqual(result, {
|
||||
'active_config_sha256': request.active_config_sha256,
|
||||
'active_secrets_sha256': request.active_secrets_sha256,
|
||||
})
|
||||
self.assertEqual(
|
||||
store['active_config_path'].read_bytes(), store['active_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
store['active_secrets_path'].read_bytes(), store['active_secrets'],
|
||||
)
|
||||
|
||||
def test_completed_publication_can_be_restored_byte_identically_once(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store)
|
||||
with host_agent_apply.HostApplySession(
|
||||
request, _Database(),
|
||||
) as session:
|
||||
session.backup()
|
||||
session.replace(self.proof(request))
|
||||
proof = self.rollback_proof(request)
|
||||
result = session.restore_backups(proof)
|
||||
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
session.restore_backups(proof)
|
||||
|
||||
self.assertEqual(raised.exception.category, 'state')
|
||||
self.assertEqual(result, {
|
||||
'active_config_sha256': request.active_config_sha256,
|
||||
'active_secrets_sha256': request.active_secrets_sha256,
|
||||
})
|
||||
self.assertEqual(
|
||||
store['active_config_path'].read_bytes(), store['active_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
store['active_secrets_path'].read_bytes(), store['active_secrets'],
|
||||
)
|
||||
operation_backup = store['backups'] / request.operation_id
|
||||
self.assertEqual(
|
||||
(operation_backup / 'config.yaml').read_bytes(),
|
||||
store['active_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
(operation_backup / 'secrets.yaml').read_bytes(),
|
||||
store['active_secrets'],
|
||||
)
|
||||
|
||||
def test_restore_failure_after_first_publish_records_partial_state(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store)
|
||||
with host_agent_apply.HostApplySession(
|
||||
request, _Database(),
|
||||
) as session:
|
||||
session.backup()
|
||||
session.replace(self.proof(request))
|
||||
replace = host_agent_apply.durable_replace
|
||||
rollback_replaces = 0
|
||||
|
||||
def fail_second_rollback(source, destination):
|
||||
nonlocal rollback_replaces
|
||||
if source.name.endswith('.rollback'):
|
||||
rollback_replaces += 1
|
||||
if rollback_replaces == 2:
|
||||
raise OSError('rollback detail')
|
||||
return replace(source, destination)
|
||||
|
||||
with mock.patch.object(
|
||||
host_agent_apply, 'durable_replace',
|
||||
side_effect=fail_second_rollback,
|
||||
):
|
||||
with self.assertRaises(host_agent_apply.HostApplyError):
|
||||
session.restore_backups(self.rollback_proof(request))
|
||||
|
||||
self.assertEqual(session.publication_state, 'partial')
|
||||
|
||||
self.assertEqual(
|
||||
store['active_config_path'].read_bytes(), store['active_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
store['active_secrets_path'].read_bytes(),
|
||||
store['proposed_secrets'],
|
||||
)
|
||||
operation_backup = store['backups'] / request.operation_id
|
||||
self.assertEqual(
|
||||
(operation_backup / 'config.yaml').read_bytes(),
|
||||
store['active_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
(operation_backup / 'secrets.yaml').read_bytes(),
|
||||
store['active_secrets'],
|
||||
)
|
||||
self.assertEqual(
|
||||
store['candidate_config_path'].read_bytes(),
|
||||
store['proposed_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
store['candidate_secrets_path'].read_bytes(),
|
||||
store['proposed_secrets'],
|
||||
)
|
||||
|
||||
def test_restore_refuses_unknown_active_bytes_without_overwrite(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store, 'apply-config')
|
||||
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
||||
session.backup()
|
||||
session.replace(self.proof(request))
|
||||
foreign = b'foreign active bytes'
|
||||
self.write_private(store['active_config_path'], foreign)
|
||||
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
session.restore_backups(self.rollback_proof(request))
|
||||
|
||||
self.assertEqual(raised.exception.category, 'identity')
|
||||
self.assertEqual(store['active_config_path'].read_bytes(), foreign)
|
||||
|
||||
@unittest.skipUnless(
|
||||
os.name != 'nt' and getattr(os, 'geteuid', lambda: -1)() == 0,
|
||||
'requires distinct root and runtime ownership',
|
||||
)
|
||||
def test_replay_adopts_root_owned_original_left_during_rollback(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store, 'apply-config')
|
||||
database = _Database()
|
||||
with host_agent_apply.HostApplySession(request, database) as session:
|
||||
session.backup()
|
||||
session.replace(self.proof(request))
|
||||
stage = store['active'] / (
|
||||
f'.config.yaml.{request.operation_id}.rollback'
|
||||
)
|
||||
host_agent_apply._create_owned_file(
|
||||
stage,
|
||||
store['active_config'],
|
||||
uid=host_agent_apply.HOST_ROOT_UID,
|
||||
gid=host_agent_apply.HOST_ROOT_GID,
|
||||
mode=0o600,
|
||||
expected_sha256=request.active_config_sha256,
|
||||
)
|
||||
host_agent_apply.durable_replace(
|
||||
stage, store['active_config_path'],
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
os.stat(store['active_config_path']).st_uid,
|
||||
host_agent_apply.HOST_ROOT_UID,
|
||||
)
|
||||
database.replayed = True
|
||||
with host_agent_apply.HostApplySession(request, database) as replay:
|
||||
result = replay.restore_backups(self.rollback_proof(request))
|
||||
|
||||
self.assertEqual(result, {
|
||||
'active_config_sha256': request.active_config_sha256,
|
||||
'active_secrets_sha256': request.active_secrets_sha256,
|
||||
})
|
||||
self.assertEqual(
|
||||
os.stat(store['active_config_path']).st_uid,
|
||||
host_agent_apply.HOST_RUNTIME_UID,
|
||||
)
|
||||
self.assertEqual(
|
||||
store['active_config_path'].read_bytes(), store['active_config'],
|
||||
)
|
||||
|
||||
@unittest.skipUnless(
|
||||
os.name != 'nt' and getattr(os, 'geteuid', lambda: -1)() == 0,
|
||||
'requires distinct root and runtime ownership',
|
||||
)
|
||||
def test_replay_adopts_root_owned_candidate_left_after_publication(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store, 'apply-config')
|
||||
database = _Database()
|
||||
with host_agent_apply.HostApplySession(request, database) as session:
|
||||
session.backup()
|
||||
stage = store['active'] / (
|
||||
f'.config.yaml.{request.operation_id}.stage'
|
||||
)
|
||||
host_agent_apply._create_owned_file(
|
||||
stage,
|
||||
store['proposed_config'],
|
||||
uid=host_agent_apply.HOST_ROOT_UID,
|
||||
gid=host_agent_apply.HOST_ROOT_GID,
|
||||
mode=0o600,
|
||||
expected_sha256=request.candidate_config_sha256,
|
||||
)
|
||||
host_agent_apply.durable_replace(
|
||||
stage, store['active_config_path'],
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
os.stat(store['active_config_path']).st_uid,
|
||||
host_agent_apply.HOST_ROOT_UID,
|
||||
)
|
||||
database.replayed = True
|
||||
with host_agent_apply.HostApplySession(request, database) as replay:
|
||||
replay.backup()
|
||||
result = replay.replace(self.proof(request))
|
||||
|
||||
self.assertEqual(
|
||||
os.stat(store['active_config_path']).st_uid,
|
||||
host_agent_apply.HOST_RUNTIME_UID,
|
||||
)
|
||||
self.assertEqual(
|
||||
result['active_config_sha256'], request.candidate_config_sha256,
|
||||
)
|
||||
|
||||
def test_stale_fixed_stage_is_removed_before_replacement(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store, 'apply-config')
|
||||
stage = store['active'] / (
|
||||
f'.config.yaml.{request.operation_id}.stage'
|
||||
)
|
||||
self.write_private(stage, b'incomplete prior stage')
|
||||
|
||||
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
||||
session.backup()
|
||||
session.replace(self.proof(request))
|
||||
|
||||
self.assertFalse(stage.exists())
|
||||
self.assertEqual(
|
||||
store['active_config_path'].read_bytes(), store['proposed_config'],
|
||||
)
|
||||
|
||||
def test_stage_remains_root_owned_until_publication(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store, 'apply-config')
|
||||
real_create = host_agent_apply._create_owned_file
|
||||
calls = []
|
||||
|
||||
def record(path, payload, **metadata):
|
||||
calls.append((Path(path), dict(metadata)))
|
||||
return real_create(path, payload, **metadata)
|
||||
|
||||
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
||||
session.backup()
|
||||
with mock.patch.object(
|
||||
host_agent_apply, '_create_owned_file', side_effect=record,
|
||||
):
|
||||
session.replace(self.proof(request))
|
||||
|
||||
stage_calls = [item for item in calls if item[0].suffix == '.stage']
|
||||
self.assertEqual(len(stage_calls), 1)
|
||||
self.assertEqual(stage_calls[0][1]['uid'], host_agent_apply.HOST_ROOT_UID)
|
||||
self.assertEqual(stage_calls[0][1]['gid'], host_agent_apply.HOST_ROOT_GID)
|
||||
|
||||
@unittest.skipIf(os.name == 'nt', 'POSIX displaced-inode guard')
|
||||
def test_active_write_after_revalidation_is_detected_as_partial(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store, 'apply-config')
|
||||
real_replace = host_agent_apply.durable_replace
|
||||
|
||||
def mutate_then_replace(source, destination):
|
||||
self.write_private(Path(destination), b'late concurrent write')
|
||||
real_replace(source, destination)
|
||||
|
||||
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
||||
session.backup()
|
||||
with mock.patch.object(
|
||||
host_agent_apply,
|
||||
'durable_replace',
|
||||
side_effect=mutate_then_replace,
|
||||
), self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
session.replace(self.proof(request))
|
||||
|
||||
self.assertEqual(raised.exception.category, 'partial')
|
||||
self.assertEqual(
|
||||
store['active_config_path'].read_bytes(), store['proposed_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
(
|
||||
store['backups'] / request.operation_id / 'config.yaml'
|
||||
).read_bytes(),
|
||||
store['active_config'],
|
||||
)
|
||||
|
||||
def test_existing_backup_must_be_byte_identical(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store, 'apply-config')
|
||||
operation_backup = store['backups'] / request.operation_id
|
||||
operation_backup.mkdir(mode=0o700)
|
||||
os.chmod(operation_backup, 0o700)
|
||||
self.write_private(operation_backup / 'config.yaml', b'foreign backup')
|
||||
|
||||
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
||||
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
session.backup()
|
||||
self.assertEqual(raised.exception.category, 'identity')
|
||||
self.assertEqual(
|
||||
(operation_backup / 'config.yaml').read_bytes(), b'foreign backup',
|
||||
)
|
||||
|
||||
def test_combined_second_publication_failure_is_reported_as_partial(self):
|
||||
with self.store() as store:
|
||||
request = self.request(store)
|
||||
real_replace = host_agent_apply.durable_replace
|
||||
calls = []
|
||||
|
||||
def fail_second(source, destination):
|
||||
calls.append(Path(destination).name)
|
||||
if len(calls) == 2:
|
||||
raise OSError('second publication failed')
|
||||
real_replace(source, destination)
|
||||
|
||||
with host_agent_apply.HostApplySession(request, _Database()) as session:
|
||||
session.backup()
|
||||
with mock.patch.object(
|
||||
host_agent_apply, 'durable_replace', side_effect=fail_second,
|
||||
), self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
session.replace(self.proof(request))
|
||||
|
||||
self.assertEqual(raised.exception.category, 'partial')
|
||||
self.assertEqual(store['active_config_path'].read_bytes(), store['proposed_config'])
|
||||
self.assertEqual(store['active_secrets_path'].read_bytes(), store['active_secrets'])
|
||||
operation_backup = store['backups'] / request.operation_id
|
||||
self.assertEqual(
|
||||
(operation_backup / 'config.yaml').read_bytes(), store['active_config'],
|
||||
)
|
||||
self.assertEqual(
|
||||
(operation_backup / 'secrets.yaml').read_bytes(), store['active_secrets'],
|
||||
)
|
||||
|
||||
@unittest.skipIf(os.name == 'nt', 'POSIX link metadata test')
|
||||
def test_candidate_hardlink_is_rejected_without_claimed_content_publication(self):
|
||||
with self.store(candidate_config=False) as store:
|
||||
os.link(store['active_config_path'], store['candidate_config_path'])
|
||||
request = self.request(store, 'apply-config')
|
||||
request = HostAgentRequest(
|
||||
operation_id=request.operation_id,
|
||||
action=request.action,
|
||||
active_config_sha256=request.active_config_sha256,
|
||||
active_secrets_sha256=request.active_secrets_sha256,
|
||||
candidate_config_sha256=request.active_config_sha256,
|
||||
candidate_secrets_sha256=None,
|
||||
)
|
||||
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
with host_agent_apply.HostApplySession(request, _Database()):
|
||||
pass
|
||||
self.assertEqual(raised.exception.category, 'filesystem')
|
||||
|
||||
@unittest.skipIf(os.name == 'nt', 'POSIX flock integration test')
|
||||
def test_real_filesystem_lock_excludes_concurrent_host_session(self):
|
||||
with self.store(real_lock=True) as store:
|
||||
request = self.request(store, 'restart')
|
||||
first = host_agent_apply.HostApplySession(request, _Database())
|
||||
first.__enter__()
|
||||
try:
|
||||
with self.assertRaises(host_agent_apply.HostApplyError) as raised:
|
||||
host_agent_apply.HostApplySession(
|
||||
request, _Database(),
|
||||
).__enter__()
|
||||
self.assertEqual(raised.exception.category, 'busy')
|
||||
finally:
|
||||
first.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user