Initial server source import

2026-09-30 20:30:56 +03:00
commit 170dd941b9
498 changed files with 261563 additions and 0 deletions
@@ -0,0 +1,137 @@
+import sys
+
+sys.dont_write_bytecode = True
+
+import argparse
+import os
+import sqlite3
+
+from scanner_db import ScannerDB
+
+
+TABLES = [
+    'runs',
+    'source_cycles',
+    'target_scans',
+    'target_queue',
+    'scan_publication_outbox',
+    'findings',
+    'errors',
+    'queue_snapshots',
+    'config_snapshots',
+    'package_repo_candidates',
+    'keycheck_results',
+    'keycheck_event_map',
+    'finding_uid_map',
+]
+
+
+def sqlite_connect_ro(path):
+    uri = 'file:' + os.path.abspath(path).replace('\\', '/') + '?mode=ro'
+    conn = sqlite3.connect(uri, uri=True)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def sqlite_columns(conn, table):
+    return [row['name'] for row in conn.execute(f'PRAGMA table_info({table})').fetchall()]
+
+
+def sqlite_count(conn, table):
+    return int(conn.execute(f'SELECT COUNT(*) AS count FROM {table}').fetchone()['count'])
+
+
+def sqlite_row_estimate(conn, table, exact=False):
+    if exact:
+        return sqlite_count(conn, table)
+    try:
+        row = conn.execute('SELECT seq FROM sqlite_sequence WHERE name = ?', (table,)).fetchone()
+        if row and row['seq'] is not None:
+            return int(row['seq'])
+    except sqlite3.Error:
+        pass
+    return None
+
+
+def pg_reset_identity(db, table):
+    db.conn.execute(f'''
+        SELECT setval(
+            pg_get_serial_sequence('{table}', 'id'),
+            COALESCE((SELECT MAX(id) FROM {table}), 1),
+            (SELECT MAX(id) IS NOT NULL FROM {table})
+        )
+    ''')
+
+
+def postgres_safe_value(value):
+    if isinstance(value, str) and '\x00' in value:
+        return value.replace('\x00', '\\u0000')
+    return value
+
+
+def copy_table(source, target, table, batch_size, dry_run=False, exact_counts=False):
+    columns = sqlite_columns(source, table)
+    if not columns:
+        print(f'{table}: missing or empty schema in SQLite, skipped', flush=True)
+        return 0
+    total = sqlite_row_estimate(source, table, exact=exact_counts)
+    total_label = total if total is not None else 'unknown'
+    print(f'{table}: source_rows={total_label}' + ('' if exact_counts else ' estimated'), flush=True)
+    if dry_run or total == 0:
+        return int(total or 0)
+
+    column_sql = ', '.join(columns)
+    placeholders = ', '.join('?' for _ in columns)
+    conflict_sql = ' ON CONFLICT (id) DO NOTHING' if 'id' in columns else ''
+    insert_sql = f'INSERT INTO {table} ({column_sql}) VALUES ({placeholders}){conflict_sql}'
+
+    copied = 0
+    cursor = source.execute(f'SELECT {column_sql} FROM {table} ORDER BY id' if 'id' in columns else f'SELECT {column_sql} FROM {table}')
+    while True:
+        rows = cursor.fetchmany(batch_size)
+        if not rows:
+            break
+        values = [[postgres_safe_value(row[column]) for column in columns] for row in rows]
+        if target.conn.is_postgres:
+            with target.conn._conn.cursor() as pg_cursor:
+                with pg_cursor.copy(f'COPY {table} ({column_sql}) FROM STDIN') as copy:
+                    for value_row in values:
+                        copy.write_row(value_row)
+        else:
+            for value_row in values:
+                target.conn.execute(insert_sql, value_row)
+        copied += len(rows)
+        target.conn.commit()
+        print(f'{table}: copied={copied}/{total_label}', flush=True)
+    if 'id' in columns:
+        pg_reset_identity(target, table)
+    target.conn.commit()
+    return copied
+
+
+def truncate_target(db):
+    table_sql = ', '.join(TABLES)
+    db.conn.execute(f'TRUNCATE TABLE {table_sql} RESTART IDENTITY CASCADE')
+    db.conn.commit()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Offline migrate scanner observability SQLite DB to PostgreSQL.')
+    parser.add_argument('--sqlite', required=True, help='Path to scanner_active.db or archived SQLite DB')
+    parser.add_argument('--db-url', default=os.getenv('SCANNER_DB_URL') or os.getenv('DATABASE_URL'), help='PostgreSQL DSN')
+    parser.add_argument('--batch-size', type=int, default=1000)
+    parser.add_argument('--exact-counts', action='store_true', help='Use exact COUNT(*) per table; slow on multi-GB SQLite files')
+    parser.add_argument('--truncate', action='store_true', help='Delete existing Postgres observability rows before import')
+    parser.add_argument('--dry-run', action='store_true')
+    return parser.parse_args()
+
+
+def main():
+    raise SystemExit(
+        'migrate_observability_db.py is retired for PostgreSQL. Use migrate_runtime_safety.py '
+        '--apply --sources-stopped with the bound cluster; perform legacy data import only with a reviewed, cluster-bound tool.'
+    )
+
+
+if __name__ == '__main__':
+    sys.exit(main())