From 76d36e5c35e6c020c6d6e1b9cb42d2d66f483b2c Mon Sep 17 00:00:00 2001 From: dev Date: Sun, 4 Oct 2026 01:36:58 +0300 Subject: [PATCH] =?UTF-8?q?feat(backup):=20=D1=84=D0=BE=D1=80=D0=BC=D0=B0?= =?UTF-8?q?=D1=82=20=D0=B1=D1=8D=D0=BA=D0=B0=D0=BF=D0=B0=20v2=20=E2=80=94?= =?UTF-8?q?=20audit/=D1=83=D0=B2=D0=B5=D0=B4=D0=BE=D0=BC=D0=BB=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=8F/=D0=B1=D0=B0=D0=BD=D1=8B,=20counts=20=D0=B8=20?= =?UTF-8?q?=D0=BE=D0=B4=D0=BD=D0=BE=D1=80=D0=B0=D0=B7=D0=BE=D0=B2=D1=8B?= =?UTF-8?q?=D0=B9=20=D1=82=D0=B8=D0=BA=D0=B5=D1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Валидация и нормализация restore вынесены из server.js в backup-restore.js, покрыты юнит-тестами backup.selftest.js (30+ проверок, без стенда). В бэкап добавлены audit_log, notifications, notification_reads, banned_ips, счётчики counts, метаданные приложения и версия формата (принимаются 1..2). Тикет бэкапа стал одноразовым: файл удаляется сразу после отдачи, uploadBackup фильтрует расширения. sessions в бэкап не входит намеренно. sweepOrphanedUploads учитывает аватары, обложки, оригиналы фото и photo_jobs — иначе restore сносил файлы сразу после восстановления. storage.getStream при STORAGE_LOCAL_FALLBACK читает локальный файл, а не S3. --- AGENTS.md | 16 +- backup-restore.js | 488 ++++++++++++++++++++++++++++++++++++++++++++ backup.selftest.js | 88 ++++++++ server.js | 498 ++++++++++----------------------------------- storage.js | 4 + 5 files changed, 698 insertions(+), 396 deletions(-) create mode 100644 backup-restore.js create mode 100644 backup.selftest.js diff --git a/AGENTS.md b/AGENTS.md index 4eec531..b4dcb5e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -155,10 +155,15 @@ This document defines how AI agents should work with the WhatIDo codebase. Follo - Cert: app generates self-signed cert at build (`certs/cert.pem`), mounted into tailscale container ### 8. Backup / Restore -- **Admin UI**: `/api/backup` (download tar.gz), `/api/restore` (upload tar.gz) +- **Admin UI**: `POST /api/backup` → тикет + `GET /api/backup/:token` (разовая ссылка, 30 мин), `POST /api/restore` (upload `.tar.gz`) - **Scripts**: `scripts/backup.sh`, `scripts/restore.sh` (host-level) -- Backup format: `data.json` (all tables) + `uploads/` directory -- Restore validates all data, resets sequences, sweeps orphans +- Формат архива: `tar.gz` с `data.json` + `uploads/`. Версия формата — `BACKUP_FORMAT_VERSION` в `backup-restore.js` (сейчас `2`), принимаются версии `1..2`; версия пишется в `data.json.version` и возвращается в ответе `POST /api/backup` и `POST /api/restore` +- `data.json` содержит `version`, `created_at`, `app` (версия/коммит), `counts` (строки по таблицам + `files`) и сами данные. Таблицы перечислены в `BACKUP_TABLES` — **при добавлении таблицы править её и в `buildBackupArchive`, и здесь** +- `sessions` в бэкап **не входит** намеренно: после restore все токены должны умереть. `audit_log`, `notifications`, `notification_reads`, `banned_ips` — входят +- Файлы: `storage.downloadAll` кладёт в архив всё, кроме регенерируемых `.thumbs/` и `.cache/`; `.originals/` (оригиналы фото до ИИ-обработки) **входят** и восстанавливаются через `uploadTree` +- Restore: валидация всего через `normalizeRestoreData` (`backup-restore.js`), транзакция с `DELETE` в FK-безопасном порядке → `INSERT` → `setval` по `BACKUP_SEQUENCE_TABLES` → файлы → `sweepOrphanedUploads()` → `loadBans()` → `invalidateAll()` +- **Колонки, которые normalizeRestoreData обязана сохранять**: `groups.deleted_at`/`purge_at`, `entries.purge_at`. Потеря `deleted_at` воскрешает мягко удалённые группы как активные — это не «мелочь», а порча данных +- `sweepOrphanedUploads()` считает ссылками фото из `entries.photo_path`/`photo_original_path`, `project_files.path`, `group_photos`, `entry_photos`, `student_photos`, `modules.photo_path`, `students.photo_path`, `groups.cover_path`, `photo_jobs.before_path`/`after_path`, `settings.system_logo`. Новая колонка с путём к файлу → добавить сюда, иначе sweep снесёт файл сразу после restore --- @@ -235,6 +240,10 @@ node redis.selftest.js # unit + degradation, ne node api.smoketest.js # e2e, needs running stack docker compose exec redis redis-cli -a "$REDIS_PASSWORD" --no-auth-warning INFO docker compose stop redis && node api.smoketest.js # app must keep working in-memory + +# Backup checks +node backup.selftest.js # unit, no stack needed +curl -sk -X POST https://127.0.0.1:3443/api/backup -H "X-Auth-Token: $TOKEN" # -> counts по всем таблицам docker compose start redis # app reconnects on its own # Audit diff checks @@ -290,6 +299,7 @@ guards against. | `redis.js` | Redis abstraction: cache, counters, rate-limit store, pub/sub, in-memory fallback | | `redis.selftest.js` | Self-tests for `redis.js`, including behaviour with Redis unavailable | | `diff.js` / `diff.selftest.js` | Word-level text diff and audit change payload; self-tests | +| `backup-restore.js` / `backup.selftest.js` | Backup format version, `normalizeRestoreData` validation of restore payloads, backup table lists; self-tests | | `api.smoketest.js` | End-to-end API smoke test against a running stack | | `worker.js` | Background AI auto-check workers: entry messages, lesson-report template check, photo enhance | | `db/init.sql` | Initial schema (runs on fresh DB) | diff --git a/backup-restore.js b/backup-restore.js new file mode 100644 index 0000000..05038c3 --- /dev/null +++ b/backup-restore.js @@ -0,0 +1,488 @@ +const PHOTO_JOB_ACTIONS = new Set(['ai', 'ai_face', 'ai_upscale', 'enhance', 'restore', 'rollback']); +const LESSON_REPORT_TEXT_MAX = 5000; +const BACKUP_FORMAT_VERSION = 2; +const BACKUP_MIN_FORMAT_VERSION = 1; +const BACKUP_TABLES = [ + 'groups', 'students', 'entries', 'project_files', 'branches', 'users', 'user_branches', + 'group_photos', 'entry_photos', 'modules', 'student_photos', 'share_links', 'photo_jobs', + 'lesson_reports', 'lesson_report_versions', 'audit_log', 'notifications', + 'notification_reads', 'banned_ips', +]; +const BACKUP_SEQUENCE_TABLES = [ + 'groups', 'students', 'entries', 'project_files', 'branches', 'users', 'group_photos', + 'entry_photos', 'modules', 'student_photos', 'share_links', 'photo_jobs', 'lesson_reports', + 'lesson_report_versions', 'audit_log', 'notifications', +]; + +function isSupportedBackupVersion(data) { + if (!data || typeof data !== 'object' || !Array.isArray(data.groups)) return false; + const v = Number(data.version); + return Number.isInteger(v) && v >= BACKUP_MIN_FORMAT_VERSION && v <= BACKUP_FORMAT_VERSION; +} + +function restoredCounts(ndata) { + const out = {}; + for (const tbl of BACKUP_TABLES) out[tbl] = Array.isArray(ndata[tbl]) ? ndata[tbl].length : 0; + out.settings = Object.keys(ndata.settings || {}).length; + return out; +} + +const SAFE_NAME = /^[\w,.()-]+$/; + +function isSafeUploadPath(p) { + if (typeof p !== 'string' || !p.startsWith('/uploads/')) return false; + const name = p.slice('/uploads/'.length); + return name !== '' && !name.includes('/') && !name.includes('..') && SAFE_NAME.test(name); +} + +function reqInt(v) { + const n = Number(v); + if (!Number.isInteger(n)) throw new Error('Invalid integer'); + return n; +} + +function optInt(v, lo = -Infinity, hi = Infinity) { + if (v === null || v === undefined || v === '') return null; + const n = Number(v); + if (!Number.isInteger(n) || n < lo || n > hi) throw new Error('Invalid integer'); + return n; +} + +function reqStr(v, max) { + if (typeof v !== 'string') throw new Error('Invalid string'); + const s = v.trim(); + if (!s || s.length > max) throw new Error('Invalid string length'); + return s; +} + +function optStr(v, max) { + if (v === null || v === undefined) return null; + return reqStr(v, max); +} + +function optTs(v) { + if (v === null || v === undefined) return null; + if (typeof v !== 'string' || !/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/.test(v)) throw new Error('Invalid timestamp'); + return v; +} + +function reqTs(v) { + const s = optTs(v); + if (!s) throw new Error('Invalid timestamp'); + return s; +} + +function optJsonText(v, max) { + if (v === null || v === undefined) return null; + if (typeof v === 'object') { + if (Array.isArray(v)) throw new Error('Invalid json'); + v = JSON.stringify(v); + } + const s = String(v); + if (!s || s.length > max) throw new Error('Invalid json'); + return s; +} + +function reqIp(v) { + const s = reqStr(v, 64); + if (!/^[0-9a-fA-F:.]+$/.test(s)) throw new Error('Invalid ip'); + return s; +} + +function optTime(v) { + if (v === null || v === undefined) return null; + if (typeof v !== 'string' || !/^\d{2}:\d{2}(:\d{2})?$/.test(v)) throw new Error('Invalid time'); + return v; +} + +function optDate(v) { + if (v === null || v === undefined) return null; + if (typeof v !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(v)) throw new Error('Invalid date'); + return v; +} + +function reqDate(v) { + const s = optDate(v); + if (!s) throw new Error('Invalid date'); + return s; +} + +function optBool(v) { + if (v === null || v === undefined) return null; + return !!v; +} + +function reqToken(v) { + if (typeof v !== 'string' || !/^[0-9a-f]{16,64}$/.test(v)) throw new Error('Invalid token'); + return v; +} + +function reqUploadPath(v, max) { + if (typeof v !== 'string' || v.length > max) throw new Error('Invalid path'); + if (!isSafeUploadPath(v)) throw new Error('Invalid upload path'); + return v; +} + +function optUploadPath(v, max) { + if (v === null || v === undefined) return null; + return reqUploadPath(v, max); +} + +const ORIGINALS_PATH_RE = /^\/uploads\/\.originals\/[\w.,()-]+$/; + +function optOriginalsPath(v, max) { + if (v === null || v === undefined) return null; + if (typeof v !== 'string' || v.length > max || !ORIGINALS_PATH_RE.test(v)) throw new Error('Invalid originals path'); + return v; +} + +function reqPhotoRefPath(v, max) { + if (typeof v !== 'string' || v.length > max) throw new Error('Invalid photo path'); + if (isSafeUploadPath(v) || ORIGINALS_PATH_RE.test(v)) return v; + throw new Error('Invalid photo path'); +} + +function optPhotoRefPath(v, max) { + if (v === null || v === undefined) return null; + return reqPhotoRefPath(v, max); +} + +function photoRefKey(p) { + if (typeof p !== 'string') return null; + if (isSafeUploadPath(p) || ORIGINALS_PATH_RE.test(p)) return p.slice('/uploads/'.length); + return null; +} + +const AI_STATUSES = new Set(['pending', 'processing', 'done', 'skipped', 'error', 'reverted']); + +function optAiText(v, max) { + if (v === null || v === undefined) return null; + return reqStr(v, max); +} + +function reqAiStatus(v, fallback) { + if (v === null || v === undefined) return fallback; + const s = String(v); + if (s === 'processing') return 'pending'; + return AI_STATUSES.has(s) ? s : fallback; +} + +const PROFILE_HREF_RE = /^(https?:\/\/|mailto:|tel:|\/|#)/i; +const PROFILE_EMAIL_RE = /^[\w.+-]+@[\w-]+\.[\w.-]{2,}$/; + +function profText(v, max) { + if (v === null || v === undefined) return null; + if (typeof v !== 'string') throw new Error('Ожидалась строка'); + const s = v.trim(); + if (!s) return null; + if (s.length > max) throw new Error('Слишком длинное значение'); + return s; +} + +function profIcon(v) { + const s = String(v || '').trim().toLowerCase(); + return /^[a-z0-9-]{1,32}$/.test(s) ? s : 'link'; +} + +function profHref(v) { + const s = String(v || '').trim(); + if (!s || s.length > 500) return null; + return (PROFILE_HREF_RE.test(s) || PROFILE_EMAIL_RE.test(s)) ? s : null; +} + +function profList(v, max, fn) { + if (v === null || v === undefined) return []; + if (!Array.isArray(v)) throw new Error('Ожидался список'); + const out = []; + for (const item of v.slice(0, max)) { + const row = fn(item); + if (row) out.push(row); + } + return out; +} + +function sanitizeStudentProfile(raw) { + if (raw === null || raw === undefined) return null; + if (typeof raw !== 'object' || Array.isArray(raw)) throw new Error('Ожидался объект профиля'); + const out = { + role: profText(raw.role, 200), + status: profText(raw.status, 60), + status_note: profText(raw.status_note, 120), + city: profText(raw.city, 120), + mentor: profText(raw.mentor, 150), + joined: profText(raw.joined, 120), + bio: profText(raw.bio, 2000), + quote: profText(raw.quote, 300), + tags: profList(raw.tags, 20, t => profText(t, 40)), + achievements: profList(raw.achievements, 40, a => profText(a, 200)), + contacts: profList(raw.contacts, 20, c => { + if (!c || typeof c !== 'object') return null; + const label = profText(c.label, 120); + if (!label) return null; + return { icon: profIcon(c.icon), label, href: profHref(c.href) }; + }), + skills: profList(raw.skills, 80, s => { + if (!s || typeof s !== 'object') return null; + const name = profText(s.name, 120); + if (!name) return null; + const value = (s.value === null || s.value === undefined || s.value === '') ? null : optInt(s.value, 0, 100); + return { group: profText(s.group, 80) || 'Навыки', name, level: profText(s.level, 40), value }; + }), + experience: profList(raw.experience, 30, e => { + if (!e || typeof e !== 'object') return null; + const title = profText(e.title, 160); + if (!title) return null; + return { + title, + company: profText(e.company, 160), + period: profText(e.period, 80), + date: profText(e.date, 40), + badge: profText(e.badge, 40), + description: profText(e.description, 800), + tags: profList(e.tags, 10, t => profText(t, 40)), + }; + }), + education: profList(raw.education, 60, m => { + if (!m || typeof m !== 'object') return null; + const module = profText(m.module, 200); + if (!module) return null; + const progress = (m.progress === null || m.progress === undefined || m.progress === '') ? null : optInt(m.progress, 0, 100); + return { module, progress, grade: profText(m.grade, 80), teacher: profText(m.teacher, 150) }; + }), + stats: profList(raw.stats, 12, s => { + if (!s || typeof s !== 'object') return null; + const label = profText(s.label, 80); + const value = (s.value === null || s.value === undefined) ? null : String(s.value).trim().slice(0, 20); + if (!label || !value) return null; + return { + icon: profIcon(s.icon || 'star'), + value, + suffix: profText(s.suffix, 20), + label, + hint: profText(s.hint, 120), + delta: profText(s.delta, 60), + }; + }), + }; + const hasData = Object.values(out).some(v => (Array.isArray(v) ? v.length > 0 : v !== null)); + return hasData ? out : null; +} + +function normalizeRestoreData(data) { + const groups = (data.groups || []).map(x => ({ + id: reqInt(x.id), + name: reqStr(x.name, 100), + created_at: optTs(x.created_at), + day_of_week: optInt(x.day_of_week, 0, 6), + time_start: optTime(x.time_start), + time_end: optTime(x.time_end), + branch_id: optInt(x.branch_id, 0, 2147483647), + tutor_id: optInt(x.tutor_id, 0, 2147483647), + cover_path: optUploadPath(x.cover_path, 255), + deleted_at: optTs(x.deleted_at), + purge_at: optTs(x.purge_at), + })); + const students = (data.students || []).map(x => ({ + id: reqInt(x.id), + name: reqStr(x.name, 150), + created_at: optTs(x.created_at), + group_id: optInt(x.group_id, 0, 2147483647), + photo_path: optUploadPath(x.photo_path, 255), + profile: sanitizeStudentProfile(x.profile), + })); + const entries = (data.entries || []).map(x => ({ + id: reqInt(x.id), + student_name: reqStr(x.student_name, 150), + group_id: reqInt(x.group_id), + module_id: optInt(x.module_id, 0, 2147483647), + description: reqStr(x.description, 100000), + description_original: optAiText(x.description_original, 100000) ?? reqStr(x.description, 100000), + description_ai: optAiText(x.description_ai, 100000), + ai_status: reqAiStatus(x.ai_status, 'skipped'), + ai_checked_at: optTs(x.ai_checked_at), + ai_error: optAiText(x.ai_error, 500), + photo_path: optUploadPath(x.photo_path, 255), + photo_original_path: optOriginalsPath(x.photo_original_path, 255), + deleted_at: optTs(x.deleted_at), + purge_at: optTs(x.purge_at), + created_at: optTs(x.created_at), + })); + const project_files = (data.project_files || []).map(x => ({ + id: reqInt(x.id), + entry_id: optInt(x.entry_id, 0, 2147483647), + token: reqToken(x.token), + path: reqUploadPath(x.path, 255), + name: reqStr(x.name, 255), + detached_at: optTs(x.detached_at), + created_at: optTs(x.created_at), + })); + const branches = (data.branches || []).map(x => ({ + id: reqInt(x.id), + name: reqStr(x.name, 200), + address: optStr(x.address, 1000), + phone: optStr(x.phone, 50), + created_at: optTs(x.created_at), + })); + const users = (data.users || []).map(x => ({ + id: reqInt(x.id), + username: reqStr(x.username, 100), + password_hash: reqStr(x.password_hash, 255), + name: optStr(x.name, 150), + role: (x.role === 'admin' || x.role === 'tutor') ? x.role : 'tutor', + is_active: !!x.is_active, + created_at: optTs(x.created_at), + })); + const user_branches = (data.user_branches || []).map(x => ({ + user_id: reqInt(x.user_id), + branch_id: reqInt(x.branch_id), + })); + const modules = (data.modules || []).map(x => ({ + id: reqInt(x.id), + name: reqStr(x.name, 200), + lessons_count: optInt(x.lessons_count, 0, 10000) ?? 0, + is_active: x.is_active !== false, + photo_path: optUploadPath(x.photo_path, 255), + created_at: optTs(x.created_at), + })); + const entry_photos = (data.entry_photos || []).map(x => ({ + id: reqInt(x.id), + entry_id: reqInt(x.entry_id), + photo_path: reqUploadPath(x.photo_path, 255), + caption: optStr(x.caption, 10000), + sort_order: optInt(x.sort_order, -2147483648, 2147483647), + created_at: optTs(x.created_at), + })); + const student_photos = (data.student_photos || []).map(x => ({ + id: reqInt(x.id), + student_id: reqInt(x.student_id), + photo_path: reqUploadPath(x.photo_path, 255), + created_at: optTs(x.created_at), + })); + const group_photos = (data.group_photos || []).map(x => ({ + id: reqInt(x.id), + group_id: reqInt(x.group_id), + photo_path: reqUploadPath(x.photo_path, 255), + caption: optStr(x.caption, 10000), + taken_at: optDate(x.taken_at), + sort_order: optInt(x.sort_order, -2147483648, 2147483647), + created_at: optTs(x.created_at), + })); + const share_links = (data.share_links || []).map(x => ({ + id: reqInt(x.id), + token: optStr(x.token, 40), + name: reqStr(x.name, 200), + group_id: optInt(x.group_id, 0, 2147483647), + student_name: optStr(x.student_name, 150), + date_from: optDate(x.date_from), + date_to: optDate(x.date_to), + show_student_names: optBool(x.show_student_names), + expires_at: optTs(x.expires_at), + access_password_hash: optStr(x.access_password_hash, 255), + message: optStr(x.message, 2000), + link_url: optStr(x.link_url, 500), + show_student_message: optBool(x.show_student_message), + show_entry_date: optBool(x.show_entry_date), + show_group_photos: optBool(x.show_group_photos), + created_at: optTs(x.created_at), + })); + const lesson_reports = (data.lesson_reports || []).map(x => ({ + id: reqInt(x.id), + group_id: reqInt(x.group_id), + lesson_date: reqDate(x.lesson_date), + lesson_time: optTime(x.lesson_time), + text: reqStr(x.text, LESSON_REPORT_TEXT_MAX), + text_original: optStr(x.text_original, LESSON_REPORT_TEXT_MAX), + text_ai: optStr(x.text_ai, LESSON_REPORT_TEXT_MAX), + ai_status: optStr(x.ai_status, 20), + ai_checked_at: optTs(x.ai_checked_at), + ai_error: optStr(x.ai_error, 500), + author_id: optInt(x.author_id, 0, 2147483647), + branch_id: optInt(x.branch_id, 0, 2147483647), + created_at: optTs(x.created_at), + updated_at: optTs(x.updated_at), + })); + const lesson_report_versions = (data.lesson_report_versions || []).map(x => ({ + id: reqInt(x.id), + lesson_report_id: reqInt(x.lesson_report_id), + text: reqStr(x.text, LESSON_REPORT_TEXT_MAX), + source: optStr(x.source, 20), + author_id: optInt(x.author_id, 0, 2147483647), + created_at: optTs(x.created_at), + })); + const settings = {}; + for (const [k, v] of Object.entries(data.settings || {})) { + settings[reqStr(k, 100)] = reqStr(String(v), 10000); + } + const audit_log = (data.audit_log || []).map(x => ({ + id: reqInt(x.id), + user_id: optInt(x.user_id, 0, 2147483647), + action: reqStr(x.action, 100), + target: optJsonText(x.target, 200000), + ip: optStr(x.ip, 45), + created_at: optTs(x.created_at), + })); + const NOTIFICATION_LEVELS = new Set(['info', 'success', 'warning', 'critical']); + const notifications = (data.notifications || []).map(x => ({ + id: reqInt(x.id), + type: reqStr(x.type, 50), + level: (x.level && NOTIFICATION_LEVELS.has(x.level)) ? x.level : 'info', + title: reqStr(x.title, 200), + body: optStr(x.body, 2000), + link: optStr(x.link, 255), + target: optJsonText(x.target, 20000), + admin_only: !!x.admin_only, + branch_id: optInt(x.branch_id, 0, 2147483647), + created_at: optTs(x.created_at), + })); + const notification_reads = (data.notification_reads || []).map(x => ({ + user_id: reqInt(x.user_id), + notification_id: reqInt(x.notification_id), + read_at: optTs(x.read_at), + })); + const banned_ips = (data.banned_ips || []).map(x => ({ + ip: reqIp(x.ip), + reason: reqStr(x.reason, 100), + banned_until: reqTs(x.banned_until), + created_at: optTs(x.created_at), + })); + const PHOTO_JOB_STATUSES = new Set(['pending', 'processing', 'done', 'error', 'rejected']); + const photo_jobs = (data.photo_jobs || []).map(x => ({ + id: reqInt(x.id), + entry_id: reqInt(x.entry_id), + action: (x.action && PHOTO_JOB_ACTIONS.has(x.action)) ? x.action : 'ai', + params: optJsonText(x.params, 20000), + before_path: optPhotoRefPath(x.before_path, 255), + after_path: optPhotoRefPath(x.after_path, 255), + status: (x.status && PHOTO_JOB_STATUSES.has(x.status)) ? x.status : 'pending', + applied: !!x.applied, + attempts: optInt(x.attempts, 0, 2147483647) ?? 0, + error: optStr(x.error, 4000), + created_at: optTs(x.created_at), + finished_at: optTs(x.finished_at), + })); + return { groups, students, entries, project_files, settings, branches, users, user_branches, entry_photos, student_photos, group_photos, share_links, modules, photo_jobs, lesson_reports, lesson_report_versions, audit_log, notifications, notification_reads, banned_ips }; +} + +module.exports = { + PHOTO_JOB_ACTIONS, + LESSON_REPORT_TEXT_MAX, + SAFE_NAME, + isSafeUploadPath, + photoRefKey, + sanitizeStudentProfile, + reqInt, + optInt, + reqStr, + optStr, + optTs, + reqTs, + optDate, + optUploadPath, + normalizeRestoreData, + BACKUP_FORMAT_VERSION, + BACKUP_MIN_FORMAT_VERSION, + BACKUP_TABLES, + BACKUP_SEQUENCE_TABLES, + restoredCounts, + isSupportedBackupVersion, +}; diff --git a/backup.selftest.js b/backup.selftest.js new file mode 100644 index 0000000..4f54cc5 --- /dev/null +++ b/backup.selftest.js @@ -0,0 +1,88 @@ +const { + BACKUP_FORMAT_VERSION, + BACKUP_TABLES, + BACKUP_SEQUENCE_TABLES, + isSupportedBackupVersion, + restoredCounts, + isSafeUploadPath, + normalizeRestoreData, +} = require('./backup-restore'); + +let failed = 0; +function ok(label, cond, extra) { + console.log(`${cond ? 'PASS' : 'FAIL'} ${label}${extra !== undefined && !cond ? ' -> ' + JSON.stringify(extra) : ''}`); + if (!cond) failed++; +} +function throws(label, fn) { + let threw = false; + try { fn(); } catch (e) { threw = true; } + ok(label, threw); +} + +ok('формат 1 (старый архив) поддерживается', isSupportedBackupVersion({ version: 1, groups: [] })); +ok(`формат ${BACKUP_FORMAT_VERSION} поддерживается`, isSupportedBackupVersion({ version: BACKUP_FORMAT_VERSION, groups: [] })); +ok('формат 0 отклоняется', !isSupportedBackupVersion({ version: 0, groups: [] })); +ok('формат из будущего отклоняется', !isSupportedBackupVersion({ version: BACKUP_FORMAT_VERSION + 1, groups: [] })); +ok('без groups отклоняется', !isSupportedBackupVersion({ version: BACKUP_FORMAT_VERSION })); +ok('не объект отклоняется', !isSupportedBackupVersion(null)); + +ok('в бэкап входят audit_log/notifications/banned_ips', + ['audit_log', 'notifications', 'notification_reads', 'banned_ips'].every(t => BACKUP_TABLES.includes(t)), + BACKUP_TABLES); +ok('sessions не попадают в бэкап', !BACKUP_TABLES.includes('sessions')); +ok('sequences сбрасываются для audit_log и notifications', + BACKUP_SEQUENCE_TABLES.includes('audit_log') && BACKUP_SEQUENCE_TABLES.includes('notifications')); + +const counts = restoredCounts({ groups: [1, 2], settings: { a: '1', b: '2' }, audit_log: [1] }); +ok('restoredCounts считает строки и settings', counts.groups === 2 && counts.settings === 2 && counts.audit_log === 1, counts); +ok('restoredCounts для отсутствующей таблицы = 0', restoredCounts({ groups: [] }).notifications === 0); + +ok('безопасный путь загрузки принимается', isSafeUploadPath('/uploads/1700000000000-abc123.jpg')); +ok('path traversal отклоняется', !isSafeUploadPath('/uploads/../../etc/passwd')); +ok('вложенный путь отклоняется', !isSafeUploadPath('/uploads/.originals/x.jpg')); +ok('чужой префикс отклоняется', !isSafeUploadPath('/etc/passwd')); + +const base = { version: BACKUP_FORMAT_VERSION, groups: [], entries: [], users: [], branches: [] }; + +const n = normalizeRestoreData({ + ...base, + groups: [{ id: 1, name: 'G', deleted_at: '2026-01-02T03:04:05.000Z', purge_at: '2026-02-03T04:05:06.000Z' }], +}); +ok('groups.deleted_at больше не теряется', n.groups[0].deleted_at === '2026-01-02T03:04:05.000Z', n.groups[0]); +ok('groups.purge_at больше не теряется', n.groups[0].purge_at === '2026-02-03T04:05:06.000Z', n.groups[0]); + +const e = normalizeRestoreData({ + ...base, + entries: [{ id: 1, student_name: 'S', group_id: 1, description: 'd', deleted_at: '2026-01-02T03:04:05.000Z', purge_at: '2026-03-04T05:06:07.000Z' }], +}); +ok('entries.purge_at больше не теряется', e.entries[0].purge_at === '2026-03-04T05:06:07.000Z', e.entries[0]); + +const nt = normalizeRestoreData({ + ...base, + notifications: [{ id: 5, type: 'entry.new', level: 'warning', title: 'T', body: 'B', link: 'l', target: { a: 1 }, admin_only: true, branch_id: 2 }], + notification_reads: [{ user_id: 1, notification_id: 5 }], + audit_log: [{ id: 7, user_id: 1, action: 'backup.download', target: { size: 5 }, ip: '1.2.3.4' }], + banned_ips: [{ ip: '203.0.113.7', reason: 'manual', banned_until: '2026-05-05T00:00:00.000Z' }], +}); +ok('notifications нормализуются', nt.notifications[0].level === 'warning' && nt.notifications[0].title === 'T', nt.notifications[0]); +ok('notifications.target остаётся JSON-строкой', nt.notifications[0].target === '{"a":1}', nt.notifications[0].target); +ok('notification_reads нормализуются', nt.notification_reads[0].notification_id === 5); +ok('audit_log нормализуется', nt.audit_log[0].action === 'backup.download' && nt.audit_log[0].target === '{"size":5}', nt.audit_log[0]); +ok('banned_ips нормализуются', nt.banned_ips[0].ip === '203.0.113.7'); + +const badLevel = normalizeRestoreData({ ...base, notifications: [{ id: 1, type: 'x', level: 'drop-table', title: 'T' }] }); +ok('неизвестный level уведомления -> info', badLevel.notifications[0].level === 'info', badLevel.notifications[0]); + +throws('мусорный ip в banned_ips отклоняется', () => normalizeRestoreData({ ...base, banned_ips: [{ ip: 'не ip', reason: 'r', banned_until: '2026-01-01T00:00:00.000Z' }] })); +throws('пустой banned_until отклоняется', () => normalizeRestoreData({ ...base, banned_ips: [{ ip: '1.2.3.4', reason: 'r' }] })); +throws('пустой action в audit_log отклоняется', () => normalizeRestoreData({ ...base, audit_log: [{ id: 1, action: ' ' }] })); +throws('не-объект в notifications.target отклоняется', () => normalizeRestoreData({ ...base, notifications: [{ id: 1, type: 'x', title: 'T', target: [1, 2, 3] }] })); +throws('группа без id отклоняется', () => normalizeRestoreData({ ...base, groups: [{ name: 'G' }] })); +throws('путь вне uploads отклоняется', () => normalizeRestoreData({ ...base, students: [{ id: 1, name: 'S', photo_path: '/etc/passwd' }] })); +throws('notifications не массив отклоняется', () => normalizeRestoreData({ ...base, notifications: { nope: 1 } })); + +const legacy = normalizeRestoreData({ version: 1, groups: [{ id: 1, name: 'G' }] }); +ok('старый архив без новых таблиц восстанавливается', Array.isArray(legacy.audit_log) && legacy.audit_log.length === 0 && legacy.groups.length === 1, Object.keys(legacy)); + +console.log(failed ? `\n${failed} проверок провалено` : '\nBACKUP SELFTEST OK'); +process.exit(failed ? 1 : 0); \ No newline at end of file diff --git a/server.js b/server.js index da27f8a..965c0a4 100644 --- a/server.js +++ b/server.js @@ -11,6 +11,27 @@ const { createZipWriter, renderStudentReport } = require('./student-report'); const { createStorage, mimeFor } = require('./storage'); const { createRedis } = require('./redis'); const { buildEntryDiff, textDiff, normalizeEditSource, stripDiffs } = require('./diff'); +const { + PHOTO_JOB_ACTIONS, + LESSON_REPORT_TEXT_MAX, + SAFE_NAME, + isSafeUploadPath, + photoRefKey, + sanitizeStudentProfile, + reqInt, + optInt, + reqStr, + optStr, + optTs, + reqTs, + optDate, + optUploadPath, + normalizeRestoreData, + BACKUP_FORMAT_VERSION, + BACKUP_SEQUENCE_TABLES, + restoredCounts, + isSupportedBackupVersion, +} = require('./backup-restore'); const https = require('https'); const path = require('path'); @@ -1142,7 +1163,7 @@ async function removeEntryFiles(entryId) { async function sweepOrphanedUploads() { const dir = UPLOADS_DIR; try { fs.mkdirSync(dir, { recursive: true }); } catch {} - const [{ rows: photos }, { rows: files }, { rows: gphotos }, { rows: ephotos }, { rows: pendingJobs }, { rows: mphotos }, { rows: sphotos }, { rows: logos }] = await Promise.all([ + const [{ rows: photos }, { rows: files }, { rows: gphotos }, { rows: ephotos }, { rows: pendingJobs }, { rows: mphotos }, { rows: sphotos }, { rows: logos }, { rows: studentAvatars }, { rows: groupCovers }, { rows: originals }, { rows: jobBefore }] = await Promise.all([ pool.query('SELECT photo_path AS p FROM entries WHERE photo_path IS NOT NULL'), pool.query('SELECT path AS p FROM project_files'), pool.query('SELECT photo_path AS p FROM group_photos'), @@ -1151,9 +1172,13 @@ async function sweepOrphanedUploads() { pool.query('SELECT photo_path AS p FROM modules WHERE photo_path IS NOT NULL'), pool.query('SELECT photo_path AS p FROM student_photos'), pool.query(`SELECT value AS p FROM settings WHERE key = 'system_logo' AND value IS NOT NULL AND value <> ''`), + pool.query('SELECT photo_path AS p FROM students WHERE photo_path IS NOT NULL'), + pool.query('SELECT cover_path AS p FROM groups WHERE cover_path IS NOT NULL'), + pool.query('SELECT photo_original_path AS p FROM entries WHERE photo_original_path IS NOT NULL'), + pool.query('SELECT before_path AS p FROM photo_jobs WHERE before_path IS NOT NULL'), ]); const refs = new Set(); - [...photos, ...files, ...gphotos, ...ephotos, ...pendingJobs, ...mphotos, ...sphotos, ...logos].forEach(r => { + [...photos, ...files, ...gphotos, ...ephotos, ...pendingJobs, ...mphotos, ...sphotos, ...logos, ...studentAvatars, ...groupCovers, ...originals, ...jobBefore].forEach(r => { const key = storage.keyFromPath(r.p); if (key) refs.add(key); }); @@ -1960,7 +1985,6 @@ const PHOTO_AI_FACE_MODELS = ['gfpgan', 'codeformer']; const PHOTO_AI_FACE_MODES = ['off', 'face', 'all']; const PHOTO_AI_DEVICE_PREFS = ['auto', 'cuda', 'cpu']; const PHOTO_AI_DEFAULT_STRENGTH = 0.7; -const PHOTO_JOB_ACTIONS = new Set(['ai', 'ai_face', 'ai_upscale', 'enhance', 'restore', 'rollback']); const AI_MODEL = process.env.AI_MODEL || 'qwen2.5-1.5b-instruct-q4_k_m.gguf'; const AI_CALL_TIMEOUT_MS = Math.max(5000, parseInt(process.env.AI_REQUEST_TIMEOUT_MS || '120000', 10) || 120000); const AI_DEFAULT_PROMPT = process.env.AI_PROMPT || 'Ты — редактор текстов. Исправь ТОЛЬКО грамматические, орфографические и пунктуационные ошибки в тексте. Приведи к правильному регистру буквы. НЕ меняй слова, структуру предложений, стиль или смысл текста. Верни ТОЛЬКО исправленный текст без пояснений.'; @@ -2089,6 +2113,7 @@ async function aiCorrectText(text) { } const BACKUP_UPLOAD_LIMIT_MB = parseInt(process.env.BACKUP_UPLOAD_LIMIT_MB || '500', 10); +const BACKUP_ARCHIVE_EXT = /\.(?:tar\.gz|tgz|gz)$/i; const uploadBackup = multer({ storage: multer.diskStorage({ @@ -2102,385 +2127,12 @@ const uploadBackup = multer({ }, }), limits: { fileSize: BACKUP_UPLOAD_LIMIT_MB * 1024 * 1024 }, + fileFilter: (_, file, cb) => { + if (!BACKUP_ARCHIVE_EXT.test(file.originalname || '')) return cb(new Error('Not allowed extension')); + cb(null, true); + }, }); -const SAFE_NAME = /^[\w,.()-]+$/; - -function isSafeUploadPath(p) { - if (typeof p !== 'string' || !p.startsWith('/uploads/')) return false; - const name = p.slice('/uploads/'.length); - return name !== '' && !name.includes('/') && !name.includes('..') && SAFE_NAME.test(name); -} - -function reqInt(v) { - const n = Number(v); - if (!Number.isInteger(n)) throw new Error('Invalid integer'); - return n; -} - -function optInt(v, lo = -Infinity, hi = Infinity) { - if (v === null || v === undefined || v === '') return null; - const n = Number(v); - if (!Number.isInteger(n) || n < lo || n > hi) throw new Error('Invalid integer'); - return n; -} - -function reqStr(v, max) { - if (typeof v !== 'string') throw new Error('Invalid string'); - const s = v.trim(); - if (!s || s.length > max) throw new Error('Invalid string length'); - return s; -} - -function optStr(v, max) { - if (v === null || v === undefined) return null; - return reqStr(v, max); -} - -function optTs(v) { - if (v === null || v === undefined) return null; - if (typeof v !== 'string' || !/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/.test(v)) throw new Error('Invalid timestamp'); - return v; -} - -function optTime(v) { - if (v === null || v === undefined) return null; - if (typeof v !== 'string' || !/^\d{2}:\d{2}(:\d{2})?$/.test(v)) throw new Error('Invalid time'); - return v; -} - -function optDate(v) { - if (v === null || v === undefined) return null; - if (typeof v !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(v)) throw new Error('Invalid date'); - return v; -} - -function reqDate(v) { - const s = optDate(v); - if (!s) throw new Error('Invalid date'); - return s; -} - -function optBool(v) { - if (v === null || v === undefined) return null; - return !!v; -} - -function reqToken(v) { - if (typeof v !== 'string' || !/^[0-9a-f]{16,64}$/.test(v)) throw new Error('Invalid token'); - return v; -} - -function reqUploadPath(v, max) { - if (typeof v !== 'string' || v.length > max) throw new Error('Invalid path'); - if (!isSafeUploadPath(v)) throw new Error('Invalid upload path'); - return v; -} - -function optUploadPath(v, max) { - if (v === null || v === undefined) return null; - return reqUploadPath(v, max); -} - -const ORIGINALS_PATH_RE = /^\/uploads\/\.originals\/[\w.,()-]+$/; - -function optOriginalsPath(v, max) { - if (v === null || v === undefined) return null; - if (typeof v !== 'string' || v.length > max || !ORIGINALS_PATH_RE.test(v)) throw new Error('Invalid originals path'); - return v; -} - -function reqPhotoRefPath(v, max) { - if (typeof v !== 'string' || v.length > max) throw new Error('Invalid photo path'); - if (isSafeUploadPath(v) || ORIGINALS_PATH_RE.test(v)) return v; - throw new Error('Invalid photo path'); -} - -function optPhotoRefPath(v, max) { - if (v === null || v === undefined) return null; - return reqPhotoRefPath(v, max); -} - -function photoRefKey(p) { - if (typeof p !== 'string') return null; - if (isSafeUploadPath(p) || ORIGINALS_PATH_RE.test(p)) return p.slice('/uploads/'.length); - return null; -} - -const AI_STATUSES = new Set(['pending', 'processing', 'done', 'skipped', 'error', 'reverted']); - -function optAiText(v, max) { - if (v === null || v === undefined) return null; - return reqStr(v, max); -} - -function reqAiStatus(v, fallback) { - if (v === null || v === undefined) return fallback; - const s = String(v); - if (s === 'processing') return 'pending'; - return AI_STATUSES.has(s) ? s : fallback; -} - -const PROFILE_HREF_RE = /^(https?:\/\/|mailto:|tel:|\/|#)/i; -const PROFILE_EMAIL_RE = /^[\w.+-]+@[\w-]+\.[\w.-]{2,}$/; - -function profText(v, max) { - if (v === null || v === undefined) return null; - if (typeof v !== 'string') throw new Error('Ожидалась строка'); - const s = v.trim(); - if (!s) return null; - if (s.length > max) throw new Error('Слишком длинное значение'); - return s; -} - -function profIcon(v) { - const s = String(v || '').trim().toLowerCase(); - return /^[a-z0-9-]{1,32}$/.test(s) ? s : 'link'; -} - -function profHref(v) { - const s = String(v || '').trim(); - if (!s || s.length > 500) return null; - return (PROFILE_HREF_RE.test(s) || PROFILE_EMAIL_RE.test(s)) ? s : null; -} - -function profList(v, max, fn) { - if (v === null || v === undefined) return []; - if (!Array.isArray(v)) throw new Error('Ожидался список'); - const out = []; - for (const item of v.slice(0, max)) { - const row = fn(item); - if (row) out.push(row); - } - return out; -} - -function sanitizeStudentProfile(raw) { - if (raw === null || raw === undefined) return null; - if (typeof raw !== 'object' || Array.isArray(raw)) throw new Error('Ожидался объект профиля'); - const out = { - role: profText(raw.role, 200), - status: profText(raw.status, 60), - status_note: profText(raw.status_note, 120), - city: profText(raw.city, 120), - mentor: profText(raw.mentor, 150), - joined: profText(raw.joined, 120), - bio: profText(raw.bio, 2000), - quote: profText(raw.quote, 300), - tags: profList(raw.tags, 20, t => profText(t, 40)), - achievements: profList(raw.achievements, 40, a => profText(a, 200)), - contacts: profList(raw.contacts, 20, c => { - if (!c || typeof c !== 'object') return null; - const label = profText(c.label, 120); - if (!label) return null; - return { icon: profIcon(c.icon), label, href: profHref(c.href) }; - }), - skills: profList(raw.skills, 80, s => { - if (!s || typeof s !== 'object') return null; - const name = profText(s.name, 120); - if (!name) return null; - const value = (s.value === null || s.value === undefined || s.value === '') ? null : optInt(s.value, 0, 100); - return { group: profText(s.group, 80) || 'Навыки', name, level: profText(s.level, 40), value }; - }), - experience: profList(raw.experience, 30, e => { - if (!e || typeof e !== 'object') return null; - const title = profText(e.title, 160); - if (!title) return null; - return { - title, - company: profText(e.company, 160), - period: profText(e.period, 80), - date: profText(e.date, 40), - badge: profText(e.badge, 40), - description: profText(e.description, 800), - tags: profList(e.tags, 10, t => profText(t, 40)), - }; - }), - education: profList(raw.education, 60, m => { - if (!m || typeof m !== 'object') return null; - const module = profText(m.module, 200); - if (!module) return null; - const progress = (m.progress === null || m.progress === undefined || m.progress === '') ? null : optInt(m.progress, 0, 100); - return { module, progress, grade: profText(m.grade, 80), teacher: profText(m.teacher, 150) }; - }), - stats: profList(raw.stats, 12, s => { - if (!s || typeof s !== 'object') return null; - const label = profText(s.label, 80); - const value = (s.value === null || s.value === undefined) ? null : String(s.value).trim().slice(0, 20); - if (!label || !value) return null; - return { - icon: profIcon(s.icon || 'star'), - value, - suffix: profText(s.suffix, 20), - label, - hint: profText(s.hint, 120), - delta: profText(s.delta, 60), - }; - }), - }; - const hasData = Object.values(out).some(v => (Array.isArray(v) ? v.length > 0 : v !== null)); - return hasData ? out : null; -} - -function normalizeRestoreData(data) { - const groups = (data.groups || []).map(x => ({ - id: reqInt(x.id), - name: reqStr(x.name, 100), - created_at: optTs(x.created_at), - day_of_week: optInt(x.day_of_week, 0, 6), - time_start: optTime(x.time_start), - time_end: optTime(x.time_end), - branch_id: optInt(x.branch_id, 0, 2147483647), - tutor_id: optInt(x.tutor_id, 0, 2147483647), - cover_path: optUploadPath(x.cover_path, 255), - })); - const students = (data.students || []).map(x => ({ - id: reqInt(x.id), - name: reqStr(x.name, 150), - created_at: optTs(x.created_at), - group_id: optInt(x.group_id, 0, 2147483647), - photo_path: optUploadPath(x.photo_path, 255), - profile: sanitizeStudentProfile(x.profile), - })); - const entries = (data.entries || []).map(x => ({ - id: reqInt(x.id), - student_name: reqStr(x.student_name, 150), - group_id: reqInt(x.group_id), - module_id: optInt(x.module_id, 0, 2147483647), - description: reqStr(x.description, 100000), - description_original: optAiText(x.description_original, 100000) ?? reqStr(x.description, 100000), - description_ai: optAiText(x.description_ai, 100000), - ai_status: reqAiStatus(x.ai_status, 'skipped'), - ai_checked_at: optTs(x.ai_checked_at), - ai_error: optAiText(x.ai_error, 500), - photo_path: optUploadPath(x.photo_path, 255), - photo_original_path: optOriginalsPath(x.photo_original_path, 255), - deleted_at: optTs(x.deleted_at), - created_at: optTs(x.created_at), - })); - const project_files = (data.project_files || []).map(x => ({ - id: reqInt(x.id), - entry_id: optInt(x.entry_id, 0, 2147483647), - token: reqToken(x.token), - path: reqUploadPath(x.path, 255), - name: reqStr(x.name, 255), - detached_at: optTs(x.detached_at), - created_at: optTs(x.created_at), - })); - const branches = (data.branches || []).map(x => ({ - id: reqInt(x.id), - name: reqStr(x.name, 200), - address: optStr(x.address, 1000), - phone: optStr(x.phone, 50), - created_at: optTs(x.created_at), - })); - const users = (data.users || []).map(x => ({ - id: reqInt(x.id), - username: reqStr(x.username, 100), - password_hash: reqStr(x.password_hash, 255), - name: optStr(x.name, 150), - role: (x.role === 'admin' || x.role === 'tutor') ? x.role : 'tutor', - is_active: !!x.is_active, - created_at: optTs(x.created_at), - })); - const user_branches = (data.user_branches || []).map(x => ({ - user_id: reqInt(x.user_id), - branch_id: reqInt(x.branch_id), - })); - const modules = (data.modules || []).map(x => ({ - id: reqInt(x.id), - name: reqStr(x.name, 200), - lessons_count: optInt(x.lessons_count, 0, 10000) ?? 0, - is_active: x.is_active !== false, - photo_path: optUploadPath(x.photo_path, 255), - created_at: optTs(x.created_at), - })); - const entry_photos = (data.entry_photos || []).map(x => ({ - id: reqInt(x.id), - entry_id: reqInt(x.entry_id), - photo_path: reqUploadPath(x.photo_path, 255), - caption: optStr(x.caption, 10000), - sort_order: optInt(x.sort_order, -2147483648, 2147483647), - created_at: optTs(x.created_at), - })); - const student_photos = (data.student_photos || []).map(x => ({ - id: reqInt(x.id), - student_id: reqInt(x.student_id), - photo_path: reqUploadPath(x.photo_path, 255), - created_at: optTs(x.created_at), - })); - const group_photos = (data.group_photos || []).map(x => ({ - id: reqInt(x.id), - group_id: reqInt(x.group_id), - photo_path: reqUploadPath(x.photo_path, 255), - caption: optStr(x.caption, 10000), - taken_at: optDate(x.taken_at), - sort_order: optInt(x.sort_order, -2147483648, 2147483647), - created_at: optTs(x.created_at), - })); - const share_links = (data.share_links || []).map(x => ({ - id: reqInt(x.id), - token: optStr(x.token, 40), - name: reqStr(x.name, 200), - group_id: optInt(x.group_id, 0, 2147483647), - student_name: optStr(x.student_name, 150), - date_from: optDate(x.date_from), - date_to: optDate(x.date_to), - show_student_names: optBool(x.show_student_names), - expires_at: optTs(x.expires_at), - access_password_hash: optStr(x.access_password_hash, 255), - message: optStr(x.message, 2000), - link_url: optStr(x.link_url, 500), - show_student_message: optBool(x.show_student_message), - show_entry_date: optBool(x.show_entry_date), - show_group_photos: optBool(x.show_group_photos), - created_at: optTs(x.created_at), - })); - const lesson_reports = (data.lesson_reports || []).map(x => ({ - id: reqInt(x.id), - group_id: reqInt(x.group_id), - lesson_date: reqDate(x.lesson_date), - lesson_time: optTime(x.lesson_time), - text: reqStr(x.text, LESSON_REPORT_TEXT_MAX), - text_original: optStr(x.text_original, LESSON_REPORT_TEXT_MAX), - text_ai: optStr(x.text_ai, LESSON_REPORT_TEXT_MAX), - ai_status: optStr(x.ai_status, 20), - ai_checked_at: optTs(x.ai_checked_at), - ai_error: optStr(x.ai_error, 500), - author_id: optInt(x.author_id, 0, 2147483647), - branch_id: optInt(x.branch_id, 0, 2147483647), - created_at: optTs(x.created_at), - updated_at: optTs(x.updated_at), - })); - const lesson_report_versions = (data.lesson_report_versions || []).map(x => ({ - id: reqInt(x.id), - lesson_report_id: reqInt(x.lesson_report_id), - text: reqStr(x.text, LESSON_REPORT_TEXT_MAX), - source: optStr(x.source, 20), - author_id: optInt(x.author_id, 0, 2147483647), - created_at: optTs(x.created_at), - })); - const settings = {}; - for (const [k, v] of Object.entries(data.settings || {})) { - settings[reqStr(k, 100)] = reqStr(String(v), 10000); - } - const PHOTO_JOB_STATUSES = new Set(['pending', 'processing', 'done', 'error', 'rejected']); - const photo_jobs = (data.photo_jobs || []).map(x => ({ - id: reqInt(x.id), - entry_id: reqInt(x.entry_id), - action: (x.action && PHOTO_JOB_ACTIONS.has(x.action)) ? x.action : 'ai', - params: (x.params === null || x.params === undefined) ? null : (typeof x.params === 'object' ? JSON.stringify(x.params) : String(x.params)), - before_path: optPhotoRefPath(x.before_path, 255), - after_path: optPhotoRefPath(x.after_path, 255), - status: (x.status && PHOTO_JOB_STATUSES.has(x.status)) ? x.status : 'pending', - applied: !!x.applied, - attempts: optInt(x.attempts, 0, 2147483647) ?? 0, - error: optStr(x.error, 4000), - created_at: optTs(x.created_at), - finished_at: optTs(x.finished_at), - })); - return { groups, students, entries, project_files, settings, branches, users, user_branches, entry_photos, student_photos, group_photos, share_links, modules, photo_jobs, lesson_reports, lesson_report_versions }; -} const BACKUP_TTL_MS = 30 * 60 * 1000; const BACKUP_DIR = path.join(os.tmpdir(), 'wido-backups'); @@ -2513,7 +2165,7 @@ function sweepBackupStorage() { async function buildBackupArchive() { const staging = fs.mkdtempSync(path.join(os.tmpdir(), 'wido-bk-')); try { - const [g, s, e, st, pf, br, us, ub, gp, ep, md, sp, sl, pj, lr, lrv] = await Promise.all([ + const [g, s, e, st, pf, br, us, ub, gp, ep, md, sp, sl, pj, lr, lrv, al, nt, nr, bi] = await Promise.all([ pool.query('SELECT * FROM groups ORDER BY id'), pool.query('SELECT * FROM students ORDER BY id'), pool.query('SELECT * FROM entries ORDER BY id'), @@ -2530,26 +2182,46 @@ async function buildBackupArchive() { pool.query('SELECT * FROM photo_jobs ORDER BY id'), pool.query('SELECT * FROM lesson_reports ORDER BY id'), pool.query('SELECT * FROM lesson_report_versions ORDER BY id'), + pool.query('SELECT * FROM audit_log ORDER BY id'), + pool.query('SELECT * FROM notifications ORDER BY id'), + pool.query('SELECT * FROM notification_reads ORDER BY user_id, notification_id'), + pool.query('SELECT * FROM banned_ips ORDER BY ip'), ]); const settings = {}; st.rows.forEach(r => { settings[r.key] = r.value; }); - const payload = { version: 1, created_at: new Date().toISOString(), groups: g.rows, students: s.rows, entries: e.rows, settings, project_files: pf.rows, branches: br.rows, users: us.rows, user_branches: ub.rows, group_photos: gp.rows, entry_photos: ep.rows, modules: md.rows, student_photos: sp.rows, share_links: sl.rows, photo_jobs: pj.rows, lesson_reports: lr.rows, lesson_report_versions: lrv.rows }; + const payload = { + version: BACKUP_FORMAT_VERSION, + created_at: new Date().toISOString(), + app: { version: getAppMeta().version, commit: getAppMeta().commit }, + counts: { + groups: g.rowCount, students: s.rowCount, entries: e.rowCount, settings: st.rowCount, + project_files: pf.rowCount, branches: br.rowCount, users: us.rowCount, user_branches: ub.rowCount, + group_photos: gp.rowCount, entry_photos: ep.rowCount, modules: md.rowCount, student_photos: sp.rowCount, + share_links: sl.rowCount, photo_jobs: pj.rowCount, lesson_reports: lr.rowCount, + lesson_report_versions: lrv.rowCount, audit_log: al.rowCount, notifications: nt.rowCount, + notification_reads: nr.rowCount, banned_ips: bi.rowCount, + }, + groups: g.rows, students: s.rows, entries: e.rows, settings, project_files: pf.rows, branches: br.rows, users: us.rows, user_branches: ub.rows, group_photos: gp.rows, entry_photos: ep.rows, modules: md.rows, student_photos: sp.rows, share_links: sl.rows, photo_jobs: pj.rows, lesson_reports: lr.rows, lesson_report_versions: lrv.rows, audit_log: al.rows, notifications: nt.rows, notification_reads: nr.rows, banned_ips: bi.rows, + }; + fs.writeFileSync(path.join(staging, 'data.json'), JSON.stringify(payload)); + const files = await storage.downloadAll(path.join(staging, 'uploads')); + payload.counts.files = files; fs.writeFileSync(path.join(staging, 'data.json'), JSON.stringify(payload)); - await storage.downloadAll(path.join(staging, 'uploads')); const stamp = new Date().toISOString().slice(0, 16).replace(/[:T]/g, '-'); fs.mkdirSync(BACKUP_DIR, { recursive: true }); const outPath = path.join(BACKUP_DIR, `whatido-backup-${stamp}-${crypto.randomBytes(4).toString('hex')}.tar.gz`); await tar.c({ gzip: { level: 1 }, file: outPath, cwd: staging }, ['data.json', 'uploads']); const { size } = fs.statSync(outPath); - return { file: outPath, name: `whatido-backup-${stamp}.tar.gz`, size }; + return { file: outPath, name: `whatido-backup-${stamp}.tar.gz`, size, counts: payload.counts }; } finally { fs.rmSync(staging, { recursive: true, force: true }); } } -function sendBackupArchive(res, archive) { +function sendBackupArchive(res, archive, onDone) { res.setHeader('Cache-Control', 'no-store'); res.download(archive.file, archive.name, (err) => { + if (typeof onDone === 'function') onDone(); if (err && !res.headersSent) res.status(500).json({ error: 'Не удалось отправить бэкап' }); }); } @@ -2561,7 +2233,7 @@ app.post('/api/backup', requireAdmin, async (req, res) => { const token = crypto.randomBytes(24).toString('hex'); const expiresAt = Date.now() + BACKUP_TTL_MS; backupTickets.set(token, { file: archive.file, name: archive.name, size: archive.size, expiresAt }); - await logAudit(req, 'backup.download', { size: archive.size }); + await logAudit(req, 'backup.download', { size: archive.size, counts: archive.counts }); await pushNotification({ type: 'backup.create', title: 'Создан архив бэкапа', @@ -2574,6 +2246,8 @@ app.post('/api/backup', requireAdmin, async (req, res) => { filename: archive.name, size: archive.size, expires_at: new Date(expiresAt).toISOString(), + counts: archive.counts, + format_version: BACKUP_FORMAT_VERSION, }); } catch (err) { console.error(err); @@ -2595,13 +2269,18 @@ app.get('/api/backup/:token', apiLimiter, (req, res) => { backupTickets.delete(token); return res.status(410).json({ error: 'Файл бэкапа больше недоступен. Сформируйте архив заново.' }); } - sendBackupArchive(res, ticket); + backupTickets.delete(token); + sendBackupArchive(res, ticket, () => { + try { fs.rmSync(ticket.file, { force: true }); } catch (e) { + console.error('backup cleanup failed:', e.message); + } + }); }); app.get('/api/backup', requireAdmin, async (req, res) => { try { const archive = await buildBackupArchive(); - await logAudit(req, 'backup.download', { size: archive.size }); + await logAudit(req, 'backup.download', { size: archive.size, counts: archive.counts }); await pushNotification({ type: 'backup.create', title: 'Создан архив бэкапа', @@ -2673,7 +2352,8 @@ app.post('/api/restore', requireAdmin, uploadBackup.single('backup'), async (req cleanupUpload(req); return res.status(400).json({ error: 'Неверный файл бэкапа' }); } - if (!data || data.version !== 1 || !Array.isArray(data.groups)) { + const formatVersion = Number(data && data.version); + if (!isSupportedBackupVersion(data)) { fs.rmSync(staging, { recursive: true, force: true }); cleanupUpload(req); return res.status(400).json({ error: 'Неверный формат бэкапа' }); @@ -2689,6 +2369,9 @@ app.post('/api/restore', requireAdmin, uploadBackup.single('backup'), async (req const client = await pool.connect(); try { await client.query('BEGIN'); + await client.query('DELETE FROM notification_reads'); + await client.query('DELETE FROM notifications'); + await client.query('DELETE FROM banned_ips'); await client.query('DELETE FROM project_files'); await client.query('DELETE FROM lesson_report_versions'); await client.query('DELETE FROM lesson_reports'); @@ -2699,6 +2382,7 @@ app.post('/api/restore', requireAdmin, uploadBackup.single('backup'), async (req await client.query('DELETE FROM groups'); await client.query('DELETE FROM user_branches'); await client.query('DELETE FROM sessions'); + await client.query('DELETE FROM audit_log'); await client.query('DELETE FROM users'); await client.query('DELETE FROM branches'); for (const x of ndata.branches) { @@ -2799,13 +2483,41 @@ app.post('/api/restore', requireAdmin, uploadBackup.single('backup'), async (req [x.id, x.student_id, x.photo_path, x.created_at] ); } + for (const x of ndata.notifications) { + const okBranch = x.branch_id == null || (await client.query('SELECT 1 FROM branches WHERE id = $1', [x.branch_id])).rowCount; + await client.query( + 'INSERT INTO notifications (id, type, level, title, body, link, target, admin_only, branch_id, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)', + [x.id, x.type, x.level, x.title, x.body, x.link, x.target, x.admin_only, okBranch ? x.branch_id : null, x.created_at] + ); + } + for (const x of ndata.notification_reads) { + const ex = await client.query('SELECT 1 FROM notifications WHERE id = $1', [x.notification_id]); + if (!ex.rowCount) continue; + await client.query( + 'INSERT INTO notification_reads (user_id, notification_id, read_at) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING', + [x.user_id, x.notification_id, x.read_at] + ); + } + for (const x of ndata.audit_log) { + const okUser = x.user_id == null || (await client.query('SELECT 1 FROM users WHERE id = $1', [x.user_id])).rowCount; + await client.query( + 'INSERT INTO audit_log (id, user_id, action, target, ip, created_at) VALUES ($1,$2,$3,$4,$5,$6)', + [x.id, okUser ? x.user_id : null, x.action, x.target, x.ip, x.created_at] + ); + } + for (const x of ndata.banned_ips) { + await client.query( + 'INSERT INTO banned_ips (ip, reason, banned_until, created_at) VALUES ($1,$2,$3,$4) ON CONFLICT (ip) DO UPDATE SET reason = EXCLUDED.reason, banned_until = EXCLUDED.banned_until', + [x.ip, x.reason, x.banned_until, x.created_at] + ); + } for (const [k, v] of Object.entries(ndata.settings)) { await client.query( 'INSERT INTO settings (key, value) VALUES ($1,$2) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value', [k, String(v ?? '')] ); } - for (const tbl of ['groups', 'students', 'entries', 'project_files', 'branches', 'users', 'group_photos', 'entry_photos', 'modules', 'student_photos', 'share_links', 'photo_jobs', 'lesson_reports', 'lesson_report_versions']) { + for (const tbl of BACKUP_SEQUENCE_TABLES) { const r = await client.query('SELECT COALESCE(MAX(id), 1) AS m FROM ' + tbl); await client.query('SELECT setval(pg_get_serial_sequence($1, $2), $3)', [tbl, 'id', r.rows[0].m]); } @@ -2831,8 +2543,9 @@ app.post('/api/restore', requireAdmin, uploadBackup.single('backup'), async (req fs.rmSync(staging, { recursive: true, force: true }); cleanupUpload(req); await sweepOrphanedUploads().catch(err => console.error('Upload sweep:', err)); + await loadBans().catch(err => console.error('Bans reload:', err)); await ensureFirstAdmin().catch(err => console.error('First admin:', err)); - await logAudit(req, 'backup.restore', {}); + await logAudit(req, 'backup.restore', { format_version: formatVersion }); await pushNotification({ type: 'backup.restore', title: 'Восстановление из бэкапа завершено', @@ -2841,7 +2554,7 @@ app.post('/api/restore', requireAdmin, uploadBackup.single('backup'), async (req adminOnly: true, }); invalidateAll(); - res.json({ ok: true }); + res.json({ ok: true, format_version: formatVersion, restored: restoredCounts(ndata) }); }); // --- Share links --- @@ -3752,7 +3465,6 @@ app.delete('/api/modules/:id/photo', requireAdmin, async (req, res) => { }); // --- Lesson reports (отчёт о занятии) --- -const LESSON_REPORT_TEXT_MAX = 5000; const LESSON_REPORT_LIST_TTL_MS = 30 * 1000; const LESSON_AI_VERSION_LIMIT = 50; const LESSON_AI_DEFAULT_PROMPT = [ diff --git a/storage.js b/storage.js index 4c33539..9293810 100644 --- a/storage.js +++ b/storage.js @@ -250,6 +250,10 @@ function createStorage(options = {}) { if (!fp || !fs.existsSync(fp)) return null; return { stream: fs.createReadStream(fp), contentLength: fs.statSync(fp).size, contentType: mimeFor(key) }; } + if (localFallback && localExists(key)) { + const fp = localFile(key); + return { stream: fs.createReadStream(fp), contentLength: fs.statSync(fp).size, contentType: mimeFor(key) }; + } try { const out = await client.send(new GetObjectCommand({ Bucket: bucket, Key: objKey })); return { stream: out.Body, contentLength: out.ContentLength || 0, contentType: out.ContentType || mimeFor(key) };