sashatrask
428 lines
22 KiB
Python
428 lines
22 KiB
Python
import contextlib
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
from types import SimpleNamespace
|
|
from unittest import mock
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'app'))
|
|
import scanner
|
|
from test_docker_staging_bounds import command_harness
|
|
|
|
|
|
FINISHED = '{"level":"info-0","msg":"finished scanning"}'
|
|
SPACE = 'fixture/long-paths'
|
|
|
|
|
|
@pytest.fixture
|
|
def hf_scan(monkeypatch):
|
|
state = SimpleNamespace(
|
|
environment={}, stdout='', stderr=FINISHED, returncode=0,
|
|
run=mock.Mock(), filters=mock.Mock(side_effect=lambda result, *a, **kw: result),
|
|
)
|
|
state.platform = SimpleNamespace(name='nt', environ=state.environment, getenv=state.environment.get)
|
|
state.run.side_effect = lambda *a, **kw: scanner.streamed_output_from_text(
|
|
state.stdout, state.stderr, state.returncode,
|
|
)
|
|
monkeypatch.setattr(scanner, 'os', state.platform)
|
|
monkeypatch.setattr(scanner, 'get_trufflehog_cmd', lambda: 'fixture-trufflehog')
|
|
monkeypatch.setattr(scanner, 'run_command_streamed', state.run)
|
|
monkeypatch.setattr(scanner, 'apply_finding_filters', state.filters)
|
|
monkeypatch.setattr(scanner.scan_config, 'trufflehog_config', '')
|
|
return state
|
|
|
|
|
|
@pytest.mark.parametrize('platform', ['nt', 'posix'])
|
|
@pytest.mark.parametrize('token', [None, 'hf_synthetic_explicit_token'])
|
|
def test_hf_child_config_is_windows_only_and_preserves_caller_environment(hf_scan, platform, token):
|
|
state = hf_scan
|
|
state.platform.name = platform
|
|
state.environment.update({
|
|
'GIT_CONFIG_COUNT': '2',
|
|
'GIT_CONFIG_KEY_0': 'http.version', 'GIT_CONFIG_VALUE_0': 'HTTP/1.1',
|
|
'GIT_CONFIG_KEY_1': 'core.longpaths', 'GIT_CONFIG_VALUE_1': 'false',
|
|
'GIT_CONFIG_PARAMETERS': "'color.ui=never'",
|
|
'GIT_ASKPASS': 'synthetic-askpass.cmd', 'GIT_TERMINAL_PROMPT': '0',
|
|
'TRUF_GIT_TOKEN': 'synthetic-git-token', 'TRUF_GIT_USERNAME': 'fixture-user',
|
|
'HUGGINGFACE_TOKEN': 'synthetic-inherited-hf', 'HF_TOKEN': 'synthetic-inherited-hf-alias',
|
|
'TRUF_SUPERVISOR_TOKEN': 'synthetic-authority', 'UNRELATED_SETTING': 'kept',
|
|
})
|
|
before = dict(state.environment)
|
|
state.stdout = json.dumps({'DetectorName': 'OfflineFixture', 'Raw': 'synthetic-finding'})
|
|
result = scanner.scan_huggingface_space(
|
|
SPACE, 23, 'FixtureDetector', 'ExcludedFixture', True, 'fixture-policy.yaml', token,
|
|
)
|
|
command, timeout, environment = state.run.call_args.args
|
|
assert command == [
|
|
'fixture-trufflehog', 'huggingface', '--space', SPACE, '--json', '--no-update',
|
|
'--config', 'fixture-policy.yaml', '--include-detectors', 'FixtureDetector',
|
|
'--exclude-detectors', 'ExcludedFixture', '--no-verification',
|
|
]
|
|
expected = dict(before)
|
|
if token:
|
|
expected.update(HUGGINGFACE_TOKEN=token, HF_TOKEN=token)
|
|
assert token not in command
|
|
if platform == 'nt':
|
|
expected.update(GIT_CONFIG_COUNT='3', GIT_CONFIG_KEY_2='core.longpaths', GIT_CONFIG_VALUE_2='true')
|
|
assert environment == expected
|
|
assert environment is not state.environment
|
|
assert state.environment == before
|
|
assert timeout == 23
|
|
assert state.run.call_args.kwargs == {}
|
|
assert result['findings'] == [{'DetectorName': 'OfflineFixture', 'Raw': 'synthetic-finding'}]
|
|
assert result['errors'] == []
|
|
assert result['scan_meta']['trufflehog_finished']
|
|
state.filters.assert_called_once_with(result, SPACE)
|
|
|
|
|
|
@pytest.mark.parametrize('count', [None, '', '0', '002', '255'])
|
|
def test_hf_appends_one_bounded_git_config_entry(hf_scan, count):
|
|
environment = hf_scan.environment
|
|
if count is not None:
|
|
environment['GIT_CONFIG_COUNT'] = count
|
|
existing = int(count or '0')
|
|
for index in range(existing):
|
|
environment[f'GIT_CONFIG_KEY_{index}'] = f'fixture.value{index}'
|
|
environment[f'GIT_CONFIG_VALUE_{index}'] = str(index)
|
|
before = dict(environment)
|
|
scanner.scan_huggingface_space(SPACE)
|
|
child = hf_scan.run.call_args.args[2]
|
|
assert child['GIT_CONFIG_COUNT'] == str(existing + 1)
|
|
assert child[f'GIT_CONFIG_KEY_{existing}'] == 'core.longpaths'
|
|
assert child[f'GIT_CONFIG_VALUE_{existing}'] == 'true'
|
|
assert all(child[key] == value for key, value in before.items() if key != 'GIT_CONFIG_COUNT')
|
|
assert environment == before
|
|
|
|
|
|
@pytest.mark.parametrize('count', ['-1', 'one', '1.0', ' 1', '256', '999', '9' * 5000, '\u0661'],
|
|
ids=['negative', 'text', 'fraction', 'space', 'over-limit', 'large', 'oversized', 'unicode'])
|
|
def test_hf_rejects_invalid_or_excessive_git_config_count_before_launch(hf_scan, count):
|
|
hf_scan.environment['GIT_CONFIG_COUNT'] = count
|
|
with pytest.raises(ValueError, match='HuggingFace GIT_CONFIG_COUNT'):
|
|
scanner.scan_huggingface_space(SPACE)
|
|
hf_scan.run.assert_not_called()
|
|
assert hf_scan.environment['GIT_CONFIG_COUNT'] == count
|
|
|
|
|
|
@pytest.mark.parametrize('cause', [
|
|
"error: unable to create file fixture.txt: Filename too long",
|
|
"error: invalid path 'invalid:name.txt'",
|
|
"error: invalid path 'con.txt'",
|
|
'fatal: unspecified checkout failure',
|
|
])
|
|
def test_hf_zero_exit_and_completion_never_hide_checkout_errors(hf_scan, cause):
|
|
token = 'hf_synthetic_redacted_token'
|
|
hf_scan.stderr = json.dumps({
|
|
'level': 'error', 'msg': 'error processing repository',
|
|
'error': 'error executing git clone: exit status 128, ' + cause
|
|
+ '\nwarning: Clone succeeded, but checkout failed.\n' + token,
|
|
}) + '\n' + FINISHED
|
|
result = scanner.scan_huggingface_space(SPACE, token=token)
|
|
assert len(result['errors']) == 1
|
|
assert cause in result['errors'][0]
|
|
assert token not in json.dumps(result)
|
|
assert result['error_class'] == 'trufflehog'
|
|
assert result['source_failure'] is False
|
|
assert result['retryable'] is True
|
|
assert not result.get('skipped')
|
|
assert result['scan_meta']['trufflehog_returncode'] == 0
|
|
assert result['scan_meta']['trufflehog_finished'] is True
|
|
assert 'git_checkout_recovery' not in result['scan_meta']
|
|
hf_scan.run.assert_called_once()
|
|
|
|
|
|
@pytest.mark.parametrize('stderr,returncode', [(FINISHED, 1), ('', 1), ('Command timed out', -1)])
|
|
def test_hf_keeps_nonzero_exit_and_timeout_errors(hf_scan, stderr, returncode):
|
|
hf_scan.stderr, hf_scan.returncode = stderr, returncode
|
|
result = scanner.scan_huggingface_space(SPACE)
|
|
assert result['errors']
|
|
assert result['scan_meta']['trufflehog_returncode'] == returncode
|
|
assert result['scan_meta']['trufflehog_finished'] == (stderr == FINISHED)
|
|
hf_scan.run.assert_called_once()
|
|
|
|
|
|
@pytest.mark.skipif(os.name != 'nt', reason='Windows owned-command environment contract')
|
|
def test_hf_config_survives_runner_filter_without_losing_askpass_or_authority(command_harness, monkeypatch):
|
|
state = command_harness
|
|
state.completed = True
|
|
environment = {
|
|
'GIT_CONFIG_COUNT': '1', 'GIT_CONFIG_KEY_0': 'core.protectNTFS', 'GIT_CONFIG_VALUE_0': 'true',
|
|
'TRUF_GIT_TOKEN': 'synthetic-git-token', 'TRUF_GIT_USERNAME': 'fixture-user',
|
|
'TRUF_SUPERVISOR_TOKEN': 'synthetic-authority', 'TRUF_POSTGRES_PASSWORD': 'synthetic-password',
|
|
'TRUF_MANAGED_POSTGRES_DSN': 'synthetic-dsn',
|
|
}
|
|
before = dict(environment)
|
|
guard = mock.Mock()
|
|
monkeypatch.setattr(scanner.os, 'environ', environment)
|
|
monkeypatch.setattr(scanner, 'require_trufflehog_launch_authority', guard)
|
|
monkeypatch.setattr(scanner, 'scoped_scan_slot_lease', lambda: (True, state.slot))
|
|
monkeypatch.setattr(scanner, 'harden_private_file', lambda path: None)
|
|
scanner.scan_huggingface_space(SPACE, token='hf_synthetic_token')
|
|
child = state.options['env']
|
|
assert child['GIT_CONFIG_COUNT'] == '2'
|
|
assert child['GIT_CONFIG_KEY_0'] == 'core.protectNTFS'
|
|
assert child['GIT_CONFIG_VALUE_0'] == 'true'
|
|
assert child['GIT_CONFIG_KEY_1'] == 'core.longpaths'
|
|
assert child['GIT_CONFIG_VALUE_1'] == 'true'
|
|
assert child['HF_TOKEN'] == child['HUGGINGFACE_TOKEN'] == 'hf_synthetic_token'
|
|
assert child['TRUF_GIT_TOKEN'] == 'synthetic-git-token'
|
|
assert child['TRUF_GIT_USERNAME'] == 'fixture-user'
|
|
assert child['GIT_ASKPASS'] == str(state.command_dir / 'git-askpass.cmd')
|
|
assert '%TRUF_GIT_TOKEN%' in (state.command_dir / 'git-askpass.cmd').read_text(encoding='ascii')
|
|
assert child['GIT_TERMINAL_PROMPT'] == '0'
|
|
assert child['TEMP'] == child['TMP'] == child['TMPDIR'] == str(state.command_dir)
|
|
assert not {'TRUF_SUPERVISOR_TOKEN', 'TRUF_POSTGRES_PASSWORD', 'TRUF_MANAGED_POSTGRES_DSN'} & child.keys()
|
|
assert environment == before
|
|
assert state.options['job_memory_limit_bytes'] == 4 * 1024 ** 3
|
|
guard.assert_called_once()
|
|
assert guard.call_args.args[0][1:4] == ['huggingface', '--space', SPACE]
|
|
state.slot.set_child_pid.assert_called_once_with(41)
|
|
state.slot.release.assert_not_called()
|
|
|
|
|
|
def test_hf_does_not_bypass_launch_authority(command_harness, monkeypatch):
|
|
launch = mock.Mock(side_effect=AssertionError('must not launch without authority'))
|
|
monkeypatch.setattr(scanner, 'OwnedProcess', launch)
|
|
monkeypatch.setattr(scanner, 'require_trufflehog_launch_authority',
|
|
mock.Mock(side_effect=RuntimeError('fixture authority denied')))
|
|
with pytest.raises(RuntimeError, match='fixture authority denied'):
|
|
scanner.scan_huggingface_space(SPACE)
|
|
launch.assert_not_called()
|
|
|
|
|
|
@pytest.fixture
|
|
def windows_git_fixture():
|
|
git_executable = shutil.which('git')
|
|
if os.name != 'nt' or not git_executable:
|
|
pytest.skip('requires installed Windows Git')
|
|
# Python cleanup needs extended paths too; Git below still receives ordinary absolute paths.
|
|
with tempfile.TemporaryDirectory(prefix='hf-long-', dir='\\\\?\\' + str(ROOT / 'tmp')) as extended_dir:
|
|
temp_dir = extended_dir[4:]
|
|
root = Path(temp_dir)
|
|
home = root / 'home'
|
|
home.mkdir()
|
|
environment = {key: value for key, value in os.environ.items()
|
|
if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'COMSPEC', 'PATHEXT'}}
|
|
environment.update(
|
|
TMP=temp_dir, TEMP=temp_dir, TMPDIR=temp_dir, HOME=str(home), USERPROFILE=str(home),
|
|
XDG_CONFIG_HOME=str(home), GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL=os.devnull,
|
|
GIT_CONFIG_SYSTEM=os.devnull, GIT_ATTR_NOSYSTEM='1', GIT_TERMINAL_PROMPT='0',
|
|
GIT_ALLOW_PROTOCOL='file',
|
|
)
|
|
config = [('core.longpaths', 'false'), ('core.protectNTFS', 'true'),
|
|
('core.hooksPath', str(home)), ('init.templateDir', str(home))]
|
|
environment['GIT_CONFIG_COUNT'] = str(len(config))
|
|
for index, (key, value) in enumerate(config):
|
|
environment[f'GIT_CONFIG_KEY_{index}'] = key
|
|
environment[f'GIT_CONFIG_VALUE_{index}'] = value
|
|
|
|
def git(*args, env=None, data=None, check=True):
|
|
completed = subprocess.run(
|
|
[git_executable, *map(str, args)], input=data, capture_output=True,
|
|
env=environment if env is None else env, cwd=root, timeout=30,
|
|
)
|
|
if check:
|
|
assert completed.returncode == 0, completed.stderr.decode('utf-8', errors='replace')
|
|
return completed
|
|
|
|
path208 = '/'.join(['a' * 64, 'b' * 63, 'c' * 75 + '.txt'])
|
|
path240 = '/'.join(['d' * 49, 'e' * 51, 'f' * 134 + '.txt'])
|
|
deeper = '/'.join(['depth_' + 'g' * 18] * 12 + ['deep.txt'])
|
|
paths = ['README.txt', path208, path240, deeper]
|
|
markers = dict(zip(paths, [
|
|
'HFLONGPATHFIXTURE_SHORTAAA_1001', 'HFLONGPATHFIXTURE_LONGTWOA_1002',
|
|
'HFLONGPATHFIXTURE_LONGTWOB_1003', 'HFLONGPATHFIXTURE_DEEPFILE_1004',
|
|
]))
|
|
assert (len(path208), max(map(len, path208.split('/')))) == (208, 79)
|
|
assert (len(path240), max(map(len, path240.split('/')))) == (240, 138)
|
|
assert all(max(map(len, path.split('/'))) < 255 for path in paths)
|
|
repositories = {}
|
|
for name, members in [('long', paths), ('short', paths[:1])]:
|
|
repository = root / (name + '.git')
|
|
git('init', '--bare', '--quiet', f'--template={home}', repository)
|
|
stream = bytearray(b'commit refs/heads/main\ncommitter Fixture <fixture@example.test> '
|
|
b'1700000000 +0000\ndata 7\nfixture\n')
|
|
for path in members:
|
|
content = (markers[path] + '\n').encode('ascii')
|
|
stream.extend(f'M 100644 inline {json.dumps(path)}\ndata {len(content)}\n'.encode('ascii'))
|
|
stream.extend(content + b'\n')
|
|
git('-C', repository, 'fast-import', '--quiet', data=bytes(stream) + b'\ndone\n')
|
|
git('-C', repository, 'symbolic-ref', 'HEAD', 'refs/heads/main')
|
|
repositories[name] = repository
|
|
yield SimpleNamespace(root=root, home=home, env=environment, git=git, paths=paths,
|
|
markers=markers, repositories=repositories, executable=git_executable)
|
|
|
|
|
|
def hf_child_environment(environment):
|
|
with mock.patch.object(scanner.os, 'environ', environment), \
|
|
mock.patch.object(scanner, 'run_command_streamed',
|
|
return_value=scanner.streamed_output_from_text('', FINISHED, 0)) as command, \
|
|
mock.patch.object(scanner, 'apply_finding_filters', side_effect=lambda result, *a: result):
|
|
scanner.scan_huggingface_space(SPACE)
|
|
return command.call_args.args[2]
|
|
|
|
|
|
def test_native_windows_git_clone_checkout_and_deeper_files(windows_git_fixture):
|
|
fixture = windows_git_fixture
|
|
fixed_env = hf_child_environment(fixture.env)
|
|
for enabled, environment in [(False, fixture.env), (True, fixed_env)]:
|
|
destination = fixture.root / (('enabled-' if enabled else 'disabled-') + 'x' * 32)
|
|
read_root = Path('\\\\?\\' + str(destination))
|
|
assert all(len(str(destination / path)) > 260 for path in fixture.paths[1:])
|
|
clone = fixture.git('clone', '--', fixture.repositories['long'], destination, env=environment, check=False)
|
|
stderr = clone.stderr.decode('utf-8', errors='replace')
|
|
assert (read_root / 'README.txt').read_text(encoding='ascii') == fixture.markers['README.txt'] + '\n'
|
|
if enabled:
|
|
assert clone.returncode == 0, stderr
|
|
assert 'Filename too long' not in stderr
|
|
for path, marker in fixture.markers.items():
|
|
assert (read_root / path).read_text(encoding='ascii') == marker + '\n'
|
|
tracked = fixture.git('-C', destination, 'ls-files', env=environment).stdout.decode().splitlines()
|
|
assert set(tracked) == set(fixture.paths)
|
|
else:
|
|
assert clone.returncode != 0
|
|
assert 'Filename too long' in stderr
|
|
assert 'Clone succeeded, but checkout failed' in stderr
|
|
print('native clone', json.dumps({'longpaths': enabled, 'returncode': clone.returncode,
|
|
'relative_lengths': list(map(len, fixture.paths)),
|
|
'absolute_lengths': [len(str(destination / path)) for path in fixture.paths]}))
|
|
|
|
destination = fixture.root / ('checkout-' + 'y' * 32)
|
|
fixture.git('clone', '--no-checkout', '--', fixture.repositories['long'], destination)
|
|
failed = fixture.git('-C', destination, 'checkout', '--force', 'HEAD', check=False)
|
|
assert failed.returncode != 0
|
|
assert b'Filename too long' in failed.stderr
|
|
fixture.git('-C', destination, 'checkout', '--force', 'HEAD', env=fixed_env)
|
|
read_root = Path('\\\\?\\' + str(destination))
|
|
for path, marker in fixture.markers.items():
|
|
assert (read_root / path).read_text(encoding='ascii') == marker + '\n'
|
|
assert fixture.git('config', '--get', 'core.longpaths').stdout.strip() == b'false'
|
|
assert fixture.git('config', '--get', 'core.longpaths', env=fixed_env).stdout.strip() == b'true'
|
|
assert fixture.git('config', '--get', 'core.protectNTFS', env=fixed_env).stdout.strip() == b'true'
|
|
|
|
|
|
@pytest.mark.parametrize('enabled', [False, True])
|
|
def test_native_windows_git_short_normal_path_is_unchanged(windows_git_fixture, enabled):
|
|
fixture = windows_git_fixture
|
|
environment = hf_child_environment(fixture.env) if enabled else fixture.env
|
|
destination = fixture.root / 'short-checkout'
|
|
assert len(str(destination / 'README.txt')) < 260
|
|
fixture.git('clone', '--', fixture.repositories['short'], destination, env=environment)
|
|
assert (destination / 'README.txt').read_text(encoding='ascii') == fixture.markers['README.txt'] + '\n'
|
|
|
|
|
|
@pytest.mark.parametrize('enabled', [False, True])
|
|
def test_installed_hf_uses_native_git_and_reads_all_fixture_paths(windows_git_fixture, monkeypatch, enabled):
|
|
fixture = windows_git_fixture
|
|
executable = Path(r'C:\Tools\trufflehog.exe')
|
|
if not executable.is_file():
|
|
pytest.skip('requires installed TruffleHog at C:\\Tools\\trufflehog.exe')
|
|
requests = []
|
|
|
|
class FixtureAPI(BaseHTTPRequestHandler):
|
|
def do_GET(self):
|
|
requests.append(self.path)
|
|
if self.path.split('?', 1)[0] != '/api/spaces/' + SPACE:
|
|
self.send_error(404)
|
|
return
|
|
body = json.dumps({'id': SPACE, 'private': False, 'gated': False, 'disabled': False,
|
|
'siblings': [{'rfilename': path} for path in fixture.paths]}).encode('ascii')
|
|
self.send_response(200)
|
|
self.send_header('Content-Type', 'application/json')
|
|
self.send_header('Content-Length', str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
server = ThreadingHTTPServer(('127.0.0.1', 0), FixtureAPI)
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
try:
|
|
endpoint = f'http://127.0.0.1:{server.server_port}'
|
|
environment = dict(fixture.env)
|
|
count = int(environment['GIT_CONFIG_COUNT'])
|
|
for suffix in ('', '.git'):
|
|
environment[f'GIT_CONFIG_KEY_{count}'] = f'url.{fixture.repositories["long"].as_uri()}.insteadOf'
|
|
environment[f'GIT_CONFIG_VALUE_{count}'] = endpoint + '/spaces/' + SPACE + suffix
|
|
count += 1
|
|
environment['GIT_CONFIG_COUNT'] = str(count)
|
|
trace = fixture.root / 'git-trace.jsonl'
|
|
environment['GIT_TRACE2_EVENT'] = str(trace)
|
|
environment['GIT_TRACE2_CONFIG_PARAMS'] = 'core.longpaths,core.protectNTFS'
|
|
policy = fixture.root / 'marker.yaml'
|
|
policy.write_text(
|
|
'detectors:\n - name: OfflineHFLongPaths\n keywords: [HFLONGPATHFIXTURE]\n'
|
|
" regex:\n marker: 'HFLONGPATHFIXTURE_[A-Z]{8}_[0-9]{4}'\n", encoding='ascii',
|
|
)
|
|
command_root = fixture.root / 'commands'
|
|
scanner.ensure_private_directory(str(command_root))
|
|
metadata = {'code_manifest': {'executables': {
|
|
'git': {'path': fixture.executable}, 'trufflehog': {'path': str(executable)},
|
|
}, 'assets': {str(policy): {'path': str(policy)}}}}
|
|
# Only bootstrap authority/lease are synthetic; runner, argv guard and OwnedProcess stay real.
|
|
monkeypatch.setattr(scanner, 'require_active_supervisor_child', lambda **kw: metadata)
|
|
monkeypatch.setattr(scanner.os, 'environ', environment)
|
|
monkeypatch.setattr(scanner, '_runtime_initialized', True)
|
|
monkeypatch.setattr(scanner.scan_config, 'work_dir', str(command_root))
|
|
monkeypatch.setattr(scanner.scan_config, 'trufflehog_path', str(executable))
|
|
monkeypatch.setattr(scanner.scan_config, 'min_free_gb', 0)
|
|
monkeypatch.setattr(scanner.scan_config, 'trufflehog_job_memory_limit_bytes', 2 * 1024 ** 3)
|
|
slot = mock.Mock(releasable=True)
|
|
monkeypatch.setattr(scanner, 'scoped_scan_slot_lease', lambda: (True, slot))
|
|
original = scanner.run_command_streamed
|
|
|
|
@contextlib.contextmanager
|
|
def local_command(command, timeout, env):
|
|
child_env = dict(env)
|
|
if not enabled:
|
|
# Negative control: ignore the appended setting, retaining inherited core.longpaths=false.
|
|
child_env['GIT_CONFIG_COUNT'] = environment['GIT_CONFIG_COUNT']
|
|
with original([*command, '--endpoint', endpoint, '--concurrency', '1'], timeout, child_env) as output:
|
|
yield output
|
|
|
|
monkeypatch.setattr(scanner, 'run_command_streamed', local_command)
|
|
before = dict(environment)
|
|
result = scanner.scan_huggingface_space(SPACE, 60, no_verification=True, trufflehog_config=str(policy))
|
|
assert environment == before
|
|
events = [json.loads(line) for line in trace.read_text(encoding='utf-8').splitlines()]
|
|
clones = [event for event in events if event.get('event') == 'start' and 'clone' in event.get('argv', [])]
|
|
assert clones, 'installed HF must delegate cloning to native Git'
|
|
assert all('--no-checkout' not in event['argv'] for event in clones)
|
|
assert any(event.get('param') == 'core.longpaths' and event.get('value') == str(enabled).lower()
|
|
for event in events)
|
|
seen = {path for path, marker in fixture.markers.items()
|
|
if any(marker in str(finding.get('Raw', '')) + str(finding.get('RawV2', ''))
|
|
for finding in result['findings'])}
|
|
assert result['scan_meta']['trufflehog_returncode'] == 0
|
|
assert result['scan_meta']['trufflehog_finished'] is True
|
|
if enabled:
|
|
assert result['errors'] == [], result['errors']
|
|
assert seen == set(fixture.paths)
|
|
else:
|
|
assert result['errors']
|
|
assert 'Filename too long' in '\n'.join(result['errors'])
|
|
assert 'Clone succeeded, but checkout failed' in '\n'.join(result['errors'])
|
|
assert result['source_failure'] is False
|
|
assert not result.get('skipped')
|
|
slot.set_child_pid.assert_called_once()
|
|
slot.release.assert_not_called()
|
|
assert not list(command_root.iterdir()), 'owned HF command directory must be cleaned'
|
|
print('installed HF', json.dumps({'longpaths': enabled, 'native_clones': len(clones),
|
|
'finished': result['scan_meta']['trufflehog_finished'], 'returncode': 0,
|
|
'marker_files': len(seen), 'errors': len(result['errors']), 'loopback_requests': requests}))
|
|
finally:
|
|
server.shutdown()
|
|
server.server_close()
|
|
thread.join(timeout=5)
|