Initial server source import
This commit is contained in:
sashatrask
@@ -0,0 +1,187 @@
|
||||
import gzip
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
APP_DIR = ROOT / 'app'
|
||||
sys.path.insert(0, str(APP_DIR))
|
||||
|
||||
import scanner
|
||||
|
||||
|
||||
def gzip_bytes(lines):
|
||||
output = io.BytesIO()
|
||||
with gzip.GzipFile(fileobj=output, mode='wb') as archive:
|
||||
for line in lines:
|
||||
archive.write(line)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
class Response:
|
||||
status_code = 200
|
||||
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
self.headers = {'Content-Length': str(len(payload))}
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def iter_content(self, chunk_size=1024 * 1024):
|
||||
for offset in range(0, len(self.payload), max(1, chunk_size)):
|
||||
yield self.payload[offset:offset + chunk_size]
|
||||
|
||||
def close(self):
|
||||
return None
|
||||
|
||||
|
||||
class GHArchiveBoundsTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
scanner.initialize_scanner_runtime(preflight_complete=True, register_cleanup=False)
|
||||
|
||||
def test_gzip_expansion_line_and_event_bounds_fail_closed(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
path = os.path.join(temp_dir, 'bomb.json.gz')
|
||||
Path(path).write_bytes(gzip_bytes([b'x' * 4096 + b'\n']))
|
||||
with self.assertRaises(scanner.GHArchiveBoundsError):
|
||||
scanner.validate_gharchive_gzip(
|
||||
path,
|
||||
decompressed_max_bytes=1024,
|
||||
max_events=10,
|
||||
max_line_bytes=8192,
|
||||
)
|
||||
with self.assertRaises(scanner.GHArchiveBoundsError):
|
||||
scanner.validate_gharchive_gzip(
|
||||
path,
|
||||
decompressed_max_bytes=8192,
|
||||
max_events=10,
|
||||
max_line_bytes=512,
|
||||
)
|
||||
|
||||
event_path = os.path.join(temp_dir, 'events.json.gz')
|
||||
Path(event_path).write_bytes(gzip_bytes([b'{}\n', b'{}\n', b'{}\n']))
|
||||
with self.assertRaises(scanner.GHArchiveBoundsError):
|
||||
scanner.validate_gharchive_gzip(
|
||||
event_path,
|
||||
decompressed_max_bytes=8192,
|
||||
max_events=2,
|
||||
max_line_bytes=512,
|
||||
)
|
||||
|
||||
def test_twenty_four_hours_many_events_obey_tiny_cache_and_repo_bounds(self):
|
||||
events = []
|
||||
for index in range(200):
|
||||
event = {
|
||||
'type': 'PushEvent',
|
||||
'repo': {'name': f'owner/repo-{index}'},
|
||||
'payload': {'ref': 'refs/heads/main', 'commits': []},
|
||||
}
|
||||
events.append(json.dumps(event, separators=(',', ':')).encode('utf-8') + b'\n')
|
||||
payload = gzip_bytes(events)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
runtime_dir = os.path.join(temp_dir, 'runtime')
|
||||
cache_dir = os.path.join(runtime_dir, 'gharchive')
|
||||
scanner.ensure_private_directory(runtime_dir, reject_reparse=True)
|
||||
scanner.ensure_private_directory(cache_dir, reject_reparse=True)
|
||||
settings = {
|
||||
'runtime_dir': runtime_dir,
|
||||
'gharchive_cache_dir': cache_dir,
|
||||
'gharchive_cache_max_items': 2,
|
||||
'gharchive_cache_max_bytes': 1024 * 1024,
|
||||
'gharchive_cache_min_free_bytes': 0,
|
||||
'gharchive_download_max_bytes': 1024 * 1024,
|
||||
'gharchive_decompressed_max_bytes': 4 * 1024 * 1024,
|
||||
'gharchive_max_events': 1000,
|
||||
'gharchive_max_line_bytes': 64 * 1024,
|
||||
'gharchive_cache_lock_timeout_sec': 5,
|
||||
}
|
||||
with mock.patch.multiple(scanner.scan_config, **settings), \
|
||||
mock.patch.object(scanner.requests, 'request', side_effect=lambda *_args, **_kwargs: Response(payload)) as request:
|
||||
targets = scanner.fetch_github_archive_repos(
|
||||
hours_back=24,
|
||||
max_repos=3,
|
||||
event_types=['PushEvent'],
|
||||
request_timeout=1,
|
||||
archive_cache_dir=cache_dir,
|
||||
)
|
||||
usage = scanner.gharchive_cache_usage(cache_dir)
|
||||
self.assertEqual(len(targets), 3)
|
||||
self.assertEqual(request.call_count, 24)
|
||||
self.assertTrue(all(call.kwargs['proxies']['no_proxy'] == '*' for call in request.call_args_list))
|
||||
self.assertLessEqual(usage['items'], 2)
|
||||
self.assertLessEqual(usage['bytes'], 1024 * 1024)
|
||||
|
||||
def test_compressed_download_cap_leaves_no_cache_artifact(self):
|
||||
payload = b'x' * 512
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
runtime_dir = os.path.join(temp_dir, 'runtime')
|
||||
cache_dir = os.path.join(runtime_dir, 'gharchive')
|
||||
scanner.ensure_private_directory(runtime_dir, reject_reparse=True)
|
||||
scanner.ensure_private_directory(cache_dir, reject_reparse=True)
|
||||
settings = {
|
||||
'runtime_dir': runtime_dir,
|
||||
'gharchive_cache_dir': cache_dir,
|
||||
'gharchive_cache_max_items': 2,
|
||||
'gharchive_cache_max_bytes': 4096,
|
||||
'gharchive_cache_min_free_bytes': 0,
|
||||
'gharchive_download_max_bytes': 128,
|
||||
'gharchive_decompressed_max_bytes': 4096,
|
||||
'gharchive_max_events': 100,
|
||||
'gharchive_max_line_bytes': 1024,
|
||||
'gharchive_cache_lock_timeout_sec': 2,
|
||||
}
|
||||
with mock.patch.multiple(scanner.scan_config, **settings), \
|
||||
mock.patch.object(scanner.requests, 'request', return_value=Response(payload)):
|
||||
with self.assertRaises(scanner.ApiRequestError):
|
||||
scanner.cached_gharchive_hour(
|
||||
scanner.datetime(2026, 7, 19, 1, tzinfo=scanner.timezone.utc),
|
||||
cache_dir,
|
||||
request_timeout=1,
|
||||
retries=1,
|
||||
)
|
||||
self.assertEqual(list(Path(cache_dir).glob('*.json.gz')), [])
|
||||
|
||||
def test_eviction_skips_an_active_oldest_cache_entry(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
runtime_dir = os.path.join(temp_dir, 'runtime')
|
||||
cache_dir = os.path.join(runtime_dir, 'gharchive')
|
||||
scanner.ensure_private_directory(runtime_dir, reject_reparse=True)
|
||||
scanner.ensure_private_directory(cache_dir, reject_reparse=True)
|
||||
first = os.path.join(cache_dir, '2026-07-19-01.json.gz')
|
||||
hour = 2
|
||||
second = os.path.join(cache_dir, f'2026-07-19-{hour:02d}.json.gz')
|
||||
while scanner.gharchive_item_lock_path(cache_dir, first) == scanner.gharchive_item_lock_path(cache_dir, second):
|
||||
hour += 1
|
||||
second = os.path.join(cache_dir, f'2026-07-19-{hour:02d}.json.gz')
|
||||
for index, path in enumerate((first, second), 1):
|
||||
Path(path).write_bytes(gzip_bytes([b'{}\n']))
|
||||
scanner.harden_private_file(path)
|
||||
os.utime(path, (index, index))
|
||||
settings = {
|
||||
'runtime_dir': runtime_dir,
|
||||
'gharchive_cache_dir': cache_dir,
|
||||
'gharchive_cache_max_items': 2,
|
||||
'gharchive_cache_max_bytes': 1024 * 1024,
|
||||
'gharchive_cache_min_free_bytes': 0,
|
||||
}
|
||||
held = scanner.PrivateFileLock(scanner.gharchive_item_lock_path(cache_dir, first)).acquire()
|
||||
try:
|
||||
with mock.patch.multiple(scanner.scan_config, **settings):
|
||||
scanner._gharchive_evict_for_capacity(cache_dir, required_items=1)
|
||||
finally:
|
||||
held.release()
|
||||
self.assertTrue(os.path.exists(first))
|
||||
self.assertFalse(os.path.exists(second))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user