feat(storage): S3-совместимое хранилище файлов (SeaweedFS/MinIO) и миграция uploads
- storage.js: абстракция хранилища с драйверами local и s3 (AWS SDK v3), ключи объектов совпадают с текущими путями /uploads/<файл>, поэтому схема БД и URL не меняются - docker-compose.yml: сервис s3 (SeaweedFS, том s3-data, API только на loopback), переменные S3_*/STORAGE_*, restart unless-stopped для app и db - docker-compose.minio.yml: оверрайд S3-сервиса на MinIO (образ из своего зеркала) - server.js/worker.js: чтение и запись файлов только через storage (отдача /uploads/*, миниатюры, share-файлы, zip-отчёты, enhance/apply/rollback, photo-worker), автосоздание бакета, глобальная персистенция загрузок multer - бэкап/восстановление и scripts/backup.sh, restore.sh — через scripts/storage-sync.js - scripts/migrate-to-s3.js: идемпотентная миграция uploads/ в бакет (--dry-run, --verify-only, --delete-local) - админка: блок «Хранилище» в системной информации - .env.example, README.md, AGENTS.md: описание драйверов, переменных и перехода на S3
This commit is contained in:
dev
@@ -8,6 +8,8 @@ const heicConvert = require('heic-convert');
|
||||
const { createEntryAutoChecker, createPhotoEnhanceWorker } = require('./worker');
|
||||
const { createZipWriter, renderStudentReport } = require('./student-report');
|
||||
|
||||
const { createStorage, mimeFor } = require('./storage');
|
||||
|
||||
const https = require('https');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
@@ -17,6 +19,8 @@ types.setTypeParser(1082, v => v);
|
||||
|
||||
const app = express();
|
||||
const pool = new Pool({ connectionString: process.env.DATABASE_URL });
|
||||
const UPLOADS_DIR = path.join(__dirname, 'uploads');
|
||||
const storage = createStorage({ dir: UPLOADS_DIR });
|
||||
|
||||
const pgClient = require('pg').Client;
|
||||
const lister = new pgClient({ connectionString: process.env.DATABASE_URL });
|
||||
@@ -257,8 +261,27 @@ app.use(helmet({
|
||||
}));
|
||||
app.use(express.json({ limit: '1mb' }));
|
||||
app.use(ipGuard);
|
||||
const THUMBS_DIR = path.join(__dirname, 'uploads', '.thumbs');
|
||||
const ORIGINALS_DIR = path.join(__dirname, 'uploads', '.originals');
|
||||
app.use((req, res, next) => {
|
||||
if (!storage.isRemote()) return next();
|
||||
res.on('finish', () => {
|
||||
if (res.statusCode >= 400) return;
|
||||
const uploaded = [];
|
||||
if (req.file) uploaded.push(req.file);
|
||||
if (Array.isArray(req.files)) uploaded.push(...req.files);
|
||||
else if (req.files && typeof req.files === 'object') {
|
||||
for (const list of Object.values(req.files)) if (Array.isArray(list)) uploaded.push(...list);
|
||||
}
|
||||
for (const f of uploaded) {
|
||||
if (!f || !f.filename || typeof f.path !== 'string') continue;
|
||||
const abs = path.resolve(f.path);
|
||||
if (!abs.startsWith(UPLOADS_DIR + path.sep)) continue;
|
||||
storage.persist(f.filename, abs).catch(err => console.error('Upload persist failed:', f.filename, err.message));
|
||||
}
|
||||
});
|
||||
next();
|
||||
});
|
||||
const THUMBS_DIR = path.join(UPLOADS_DIR, '.thumbs');
|
||||
const ORIGINALS_DIR = path.join(UPLOADS_DIR, '.originals');
|
||||
const THUMB_WIDTH = 480;
|
||||
let sharp = null;
|
||||
try { sharp = require('sharp'); } catch {}
|
||||
@@ -266,22 +289,30 @@ if (sharp) {
|
||||
try { fs.mkdirSync(THUMBS_DIR, { recursive: true }); } catch {}
|
||||
}
|
||||
try { fs.mkdirSync(ORIGINALS_DIR, { recursive: true }); } catch {}
|
||||
try { fs.mkdirSync(storage.cacheDir, { recursive: true }); } catch {}
|
||||
|
||||
function thumbFileFor(fp) {
|
||||
const base = path.basename(fp).replace(/\.[^.]+$/, '') + '.webp';
|
||||
return path.join(THUMBS_DIR, base);
|
||||
}
|
||||
|
||||
async function sendImageThumb(res, fp) {
|
||||
function thumbUnlinkFor(key) {
|
||||
const tp = thumbFileFor(key || '');
|
||||
try { if (fs.existsSync(tp)) fs.unlinkSync(tp); } catch {}
|
||||
}
|
||||
|
||||
async function sendImageThumb(res, key) {
|
||||
const local = await storage.localize(key);
|
||||
if (!local) return res.status(404).end();
|
||||
if (!sharp) {
|
||||
res.setHeader('Cache-Control', 'public, max-age=3600');
|
||||
return res.sendFile(fp);
|
||||
return res.sendFile(local);
|
||||
}
|
||||
const tp = thumbFileFor(fp);
|
||||
const tp = thumbFileFor(key);
|
||||
try {
|
||||
if (!fs.existsSync(tp)) {
|
||||
const tmp = tp + '.' + crypto.randomBytes(4).toString('hex') + '.tmp';
|
||||
await sharp(fp).rotate().resize({ width: THUMB_WIDTH, withoutEnlargement: true }).webp({ quality: 85 }).toFile(tmp);
|
||||
await sharp(local).rotate().resize({ width: THUMB_WIDTH, withoutEnlargement: true }).webp({ quality: 85 }).toFile(tmp);
|
||||
fs.renameSync(tmp, tp);
|
||||
}
|
||||
res.setHeader('Cache-Control', 'public, max-age=31536000, immutable');
|
||||
@@ -289,24 +320,20 @@ async function sendImageThumb(res, fp) {
|
||||
} catch {
|
||||
try { if (fs.existsSync(tp)) fs.unlinkSync(tp); } catch {}
|
||||
res.setHeader('Cache-Control', 'public, max-age=3600');
|
||||
return res.sendFile(fp);
|
||||
return res.sendFile(local);
|
||||
}
|
||||
}
|
||||
|
||||
app.get('/uploads/thumb/:name', fileLimiter, async (req, res) => {
|
||||
const name = req.params.name;
|
||||
if (!/^[A-Za-z0-9._-]+$/.test(name)) return res.status(400).end();
|
||||
const fp = path.join(__dirname, 'uploads', name);
|
||||
if (!fs.existsSync(fp) || !fs.statSync(fp).isFile()) return res.status(404).end();
|
||||
return sendImageThumb(res, fp);
|
||||
return sendImageThumb(res, name);
|
||||
});
|
||||
|
||||
app.get('/uploads/.originals/:name', fileLimiter, async (req, res) => {
|
||||
const name = req.params.name;
|
||||
if (!/^[A-Za-z0-9._-]+$/.test(name)) return res.status(400).end();
|
||||
const fp = path.join(ORIGINALS_DIR, name);
|
||||
if (!fs.existsSync(fp) || !fs.statSync(fp).isFile()) return res.status(404).end();
|
||||
return sendImageThumb(res, fp);
|
||||
return sendImageThumb(res, `.originals/${name}`);
|
||||
});
|
||||
|
||||
app.use((req, res, next) => {
|
||||
@@ -317,7 +344,17 @@ app.use((req, res, next) => {
|
||||
next();
|
||||
});
|
||||
|
||||
app.use('/uploads', express.static(path.join(__dirname, 'uploads'), { maxAge: '365d', immutable: true }));
|
||||
app.get('/uploads/*', async (req, res) => {
|
||||
const key = storage.keyFromPath(req.params[0]);
|
||||
if (!key) return res.status(400).end();
|
||||
try {
|
||||
const ok = await storage.streamTo(res, key, { cacheControl: 'public, max-age=31536000, immutable' });
|
||||
if (!ok && !res.headersSent) return res.status(404).end();
|
||||
} catch (e) {
|
||||
console.error('Upload stream failed:', key, e.message);
|
||||
if (!res.headersSent) res.status(404).end();
|
||||
}
|
||||
});
|
||||
app.use('/vendor', express.static(path.join(__dirname, 'public', 'vendor'), { maxAge: '30d' }));
|
||||
app.use(express.static(path.join(__dirname, 'public')));
|
||||
|
||||
@@ -580,16 +617,12 @@ async function purgeScheduledDeletions() {
|
||||
return { entries: erefs.rowCount, groups: gcount };
|
||||
}
|
||||
|
||||
const UPLOADS_DIR = path.join(__dirname, 'uploads');
|
||||
|
||||
function safeUnlink(relPath) {
|
||||
if (!relPath || typeof relPath !== 'string') return;
|
||||
const parts = String(relPath).replace(/\\/g, '/').replace(/^\/+/, '').split('/');
|
||||
if (parts[0] === 'uploads') parts.shift();
|
||||
if (!parts.length || parts.includes('..') || parts.includes('')) return;
|
||||
const fp = path.resolve(UPLOADS_DIR, ...parts);
|
||||
if (fp === UPLOADS_DIR || !fp.startsWith(UPLOADS_DIR + path.sep)) return;
|
||||
if (fs.existsSync(fp)) fs.unlinkSync(fp);
|
||||
const key = storage.keyFromPath(relPath);
|
||||
if (!key) return;
|
||||
storage.unlinkLocalOnly(key);
|
||||
if (!storage.isRemote()) return;
|
||||
storage.del(key).catch(err => console.error('Upload delete failed:', key, err.message));
|
||||
}
|
||||
|
||||
function removeUpload(file) {
|
||||
@@ -638,8 +671,8 @@ async function removeEntryFiles(entryId) {
|
||||
}
|
||||
|
||||
async function sweepOrphanedUploads() {
|
||||
const dir = path.join(__dirname, 'uploads');
|
||||
if (!fs.existsSync(dir)) return;
|
||||
const dir = UPLOADS_DIR;
|
||||
try { fs.mkdirSync(dir, { recursive: true }); } catch {}
|
||||
const [{ rows: photos }, { rows: files }, { rows: gphotos }, { rows: ephotos }, { rows: pendingJobs }, { rows: mphotos }, { rows: sphotos }, { rows: logos }] = await Promise.all([
|
||||
pool.query('SELECT photo_path AS p FROM entries WHERE photo_path IS NOT NULL'),
|
||||
pool.query('SELECT path AS p FROM project_files'),
|
||||
@@ -651,14 +684,28 @@ async function sweepOrphanedUploads() {
|
||||
pool.query(`SELECT value AS p FROM settings WHERE key = 'system_logo' AND value IS NOT NULL AND value <> ''`),
|
||||
]);
|
||||
const refs = new Set();
|
||||
[...photos, ...files, ...gphotos, ...ephotos, ...pendingJobs, ...mphotos, ...sphotos, ...logos].forEach(r => refs.add('/' + String(r.p).replace(/^\/+/, '')));
|
||||
[...photos, ...files, ...gphotos, ...ephotos, ...pendingJobs, ...mphotos, ...sphotos, ...logos].forEach(r => {
|
||||
const key = storage.keyFromPath(r.p);
|
||||
if (key) refs.add(key);
|
||||
});
|
||||
const remoteObjects = await storage.listAll('').catch(err => {
|
||||
console.error('Storage list failed:', err.message);
|
||||
return [];
|
||||
});
|
||||
for (const obj of remoteObjects) {
|
||||
if (!storage.normalizeKey(obj.key) || obj.key.includes('/')) continue;
|
||||
if (!refs.has(obj.key)) {
|
||||
try { await storage.del(obj.key); } catch (err) { console.error('Upload delete failed:', obj.key, err.message); }
|
||||
}
|
||||
}
|
||||
for (const f of fs.readdirSync(dir)) {
|
||||
const fp = path.join(dir, f);
|
||||
if (!fs.statSync(fp).isFile()) continue;
|
||||
if (!refs.has('/uploads/' + f)) {
|
||||
if (!refs.has(f)) {
|
||||
try { fs.unlinkSync(fp); } catch {}
|
||||
}
|
||||
}
|
||||
storage.pruneCache();
|
||||
}
|
||||
|
||||
async function ensureAuditTable() {
|
||||
@@ -1704,22 +1751,7 @@ async function buildBackupArchive() {
|
||||
st.rows.forEach(r => { settings[r.key] = r.value; });
|
||||
const payload = { version: 1, created_at: new Date().toISOString(), groups: g.rows, students: s.rows, entries: e.rows, settings, project_files: pf.rows, branches: br.rows, users: us.rows, user_branches: ub.rows, group_photos: gp.rows, entry_photos: ep.rows, modules: md.rows, student_photos: sp.rows, share_links: sl.rows, photo_jobs: pj.rows };
|
||||
fs.writeFileSync(path.join(staging, 'data.json'), JSON.stringify(payload));
|
||||
fs.mkdirSync(path.join(staging, 'uploads'), { recursive: true });
|
||||
const dir = path.join(__dirname, 'uploads');
|
||||
if (fs.existsSync(dir)) {
|
||||
for (const f of fs.readdirSync(dir)) {
|
||||
const fp = path.join(dir, f);
|
||||
if (fs.statSync(fp).isFile() && SAFE_NAME.test(f)) fs.copyFileSync(fp, path.join(staging, 'uploads', f));
|
||||
}
|
||||
const orig = path.join(dir, '.originals');
|
||||
if (fs.existsSync(orig)) {
|
||||
fs.mkdirSync(path.join(staging, 'uploads', '.originals'), { recursive: true });
|
||||
for (const f of fs.readdirSync(orig)) {
|
||||
const ofp = path.join(orig, f);
|
||||
if (fs.statSync(ofp).isFile() && SAFE_NAME.test(f)) fs.copyFileSync(ofp, path.join(staging, 'uploads', '.originals', f));
|
||||
}
|
||||
}
|
||||
}
|
||||
await storage.downloadAll(path.join(staging, 'uploads'));
|
||||
const stamp = new Date().toISOString().slice(0, 16).replace(/[:T]/g, '-');
|
||||
fs.mkdirSync(BACKUP_DIR, { recursive: true });
|
||||
const outPath = path.join(BACKUP_DIR, `whatido-backup-${stamp}-${crypto.randomBytes(4).toString('hex')}.tar.gz`);
|
||||
@@ -1971,31 +2003,14 @@ app.post('/api/restore', requireAdmin, uploadBackup.single('backup'), async (req
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
const dir = path.join(__dirname, 'uploads');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.mkdirSync(UPLOADS_DIR, { recursive: true });
|
||||
if (legacyPhotos.length) {
|
||||
for (const p of legacyPhotos) {
|
||||
if (!p.path || !SAFE_NAME.test(p.path)) continue;
|
||||
fs.writeFileSync(path.join(dir, p.path), Buffer.from(p.data, 'base64'));
|
||||
await storage.put(p.path, Buffer.from(p.data, 'base64'));
|
||||
}
|
||||
} else {
|
||||
const src = path.join(staging, 'uploads');
|
||||
if (fs.existsSync(src)) {
|
||||
for (const f of fs.readdirSync(src)) {
|
||||
if (!SAFE_NAME.test(f)) continue;
|
||||
const fp = path.join(src, f);
|
||||
if (fs.statSync(fp).isFile()) fs.copyFileSync(fp, path.join(dir, f));
|
||||
}
|
||||
const orgSrc = path.join(src, '.originals');
|
||||
if (fs.existsSync(orgSrc)) {
|
||||
fs.mkdirSync(path.join(dir, '.originals'), { recursive: true });
|
||||
for (const f of fs.readdirSync(orgSrc)) {
|
||||
if (!SAFE_NAME.test(f)) continue;
|
||||
const ofp = path.join(orgSrc, f);
|
||||
if (fs.statSync(ofp).isFile()) fs.copyFileSync(ofp, path.join(dir, '.originals', f));
|
||||
}
|
||||
}
|
||||
}
|
||||
await storage.uploadTree(path.join(staging, 'uploads'));
|
||||
}
|
||||
fs.rmSync(staging, { recursive: true, force: true });
|
||||
cleanupUpload(req);
|
||||
@@ -2315,14 +2330,17 @@ app.get('/api/share/:shareToken/files/:fileToken', fileLimiter, async (req, res)
|
||||
);
|
||||
if (!rows.length) return res.status(404).json({ error: 'Not found' });
|
||||
const r = rows[0];
|
||||
const fp = path.join(__dirname, r.path);
|
||||
if (!fs.existsSync(fp)) return res.status(404).json({ error: 'File missing' });
|
||||
const key = storage.keyFromPath(r.path);
|
||||
if (!key) return res.status(404).json({ error: 'File missing' });
|
||||
if (isImageName(r.name)) {
|
||||
if (req.query.thumb) return sendImageThumb(res, fp);
|
||||
res.setHeader('Cache-Control', 'public, max-age=31536000, immutable');
|
||||
return res.sendFile(fp);
|
||||
if (req.query.thumb) return sendImageThumb(res, key);
|
||||
const ok = await storage.streamTo(res, key, { cacheControl: 'public, max-age=31536000, immutable' });
|
||||
if (!ok && !res.headersSent) return res.status(404).json({ error: 'File missing' });
|
||||
return;
|
||||
}
|
||||
if (!(await storage.streamTo(res, key, { download: true, name: r.name })) && !res.headersSent) {
|
||||
return res.status(404).json({ error: 'File missing' });
|
||||
}
|
||||
return res.download(fp, r.name);
|
||||
});
|
||||
|
||||
app.get('/s/:token', (req, res) => {
|
||||
@@ -3332,15 +3350,14 @@ app.get('/api/export/student', requireAuth, async (req, res) => {
|
||||
|
||||
const seenPhotos = new Set();
|
||||
const photosBuilt = [];
|
||||
function addPhoto(p) {
|
||||
async function addPhoto(p) {
|
||||
if (!isSafeUploadPath(p.photo_path)) return;
|
||||
const stored = p.photo_path.slice('/uploads/'.length);
|
||||
if (seenPhotos.has(stored)) return;
|
||||
seenPhotos.add(stored);
|
||||
const src = path.join(UPLOADS_DIR, stored);
|
||||
if (!fs.existsSync(src)) return;
|
||||
const data = await storage.getBuffer(stored);
|
||||
if (!data) return;
|
||||
const ts = p.created_at ? new Date(p.created_at) : new Date();
|
||||
const data = fs.readFileSync(src);
|
||||
zip.addFile('photos/' + stored, data, ts);
|
||||
photosBuilt.push({
|
||||
stored,
|
||||
@@ -3356,13 +3373,13 @@ app.get('/api/export/student', requireAuth, async (req, res) => {
|
||||
const profilePhoto = (student.profile && student.profile.photo_path) || student.photo_path;
|
||||
if (profilePhoto && isSafeUploadPath(profilePhoto)) {
|
||||
const stored = profilePhoto.slice('/uploads/'.length);
|
||||
if (fs.existsSync(path.join(UPLOADS_DIR, stored))) avatarStored = stored;
|
||||
if (await storage.exists(stored)) avatarStored = stored;
|
||||
}
|
||||
for (const p of photosRes.rows) addPhoto(p);
|
||||
for (const m of mainsRes.rows) addPhoto(m);
|
||||
for (const p of photosRes.rows) await addPhoto(p);
|
||||
for (const m of mainsRes.rows) await addPhoto(m);
|
||||
photosBuilt.sort((a, b) => new Date(b.createdAt || Date.now()) - new Date(a.createdAt || Date.now()));
|
||||
if (avatarStored) {
|
||||
if (!seenPhotos.has(avatarStored)) addPhoto({ photo_path: '/uploads/' + avatarStored, caption: 'Фото резидента', created_at: student.created_at });
|
||||
if (!seenPhotos.has(avatarStored)) await addPhoto({ photo_path: '/uploads/' + avatarStored, caption: 'Фото резидента', created_at: student.created_at });
|
||||
const idx = photosBuilt.findIndex(p => p.stored === avatarStored);
|
||||
if (idx > 0) photosBuilt.unshift(photosBuilt.splice(idx, 1)[0]);
|
||||
}
|
||||
@@ -3374,10 +3391,10 @@ app.get('/api/export/student', requireAuth, async (req, res) => {
|
||||
if (!isSafeUploadPath(gp.photo_path)) continue;
|
||||
const stored = gp.photo_path.slice('/uploads/'.length);
|
||||
if (seenPhotos.has(stored)) continue;
|
||||
const src = path.join(UPLOADS_DIR, stored);
|
||||
if (!fs.existsSync(src)) continue;
|
||||
const data = await storage.getBuffer(stored);
|
||||
if (!data) continue;
|
||||
const ts = gp.taken_at || gp.created_at || new Date();
|
||||
zip.addFile('photos/' + stored, fs.readFileSync(src), new Date(ts));
|
||||
zip.addFile('photos/' + stored, data, new Date(ts));
|
||||
seenPhotos.add(stored);
|
||||
groupPhotosBuilt.push({
|
||||
stored,
|
||||
@@ -3395,9 +3412,8 @@ app.get('/api/export/student', requireAuth, async (req, res) => {
|
||||
for (const f of filesRes.rows) {
|
||||
if (!isSafeUploadPath(f.path)) continue;
|
||||
const stored = f.path.slice('/uploads/'.length);
|
||||
const src = path.join(UPLOADS_DIR, stored);
|
||||
if (!fs.existsSync(src)) continue;
|
||||
const data = fs.readFileSync(src);
|
||||
const data = await storage.getBuffer(stored);
|
||||
if (!data) continue;
|
||||
let base = String(f.name || 'file').replace(/[\\/:*?"<>|]/g, '_').replace(/^[.\s]+/, '').slice(0, 120) || 'file';
|
||||
const ext = path.extname(base);
|
||||
const stem = ext ? base.slice(0, -ext.length) : base;
|
||||
@@ -3700,11 +3716,10 @@ app.get('/api/files', requireAuth, async (req, res) => {
|
||||
const off = parseInt(offset, 10);
|
||||
if (off > 0) { qparams.push(off); q += ` OFFSET $${qparams.length}`; }
|
||||
const { rows } = await pool.query(q, qparams);
|
||||
const files = rows.map(r => {
|
||||
let size = 0;
|
||||
try { size = fs.statSync(path.join(__dirname, r.path)).size; } catch {}
|
||||
const files = await Promise.all(rows.map(async r => {
|
||||
const size = await storage.sizeOf(r.path);
|
||||
return { id: r.id, token: r.token, name: r.name, student_name: r.student_name, group_name: r.group_name, created_at: r.created_at, size };
|
||||
});
|
||||
}));
|
||||
res.json({ files, total });
|
||||
});
|
||||
|
||||
@@ -3728,11 +3743,10 @@ app.get('/api/files/detached', requireAdmin, async (req, res) => {
|
||||
const off = parseInt(offset, 10);
|
||||
if (off > 0) { qparams.push(off); q += ` OFFSET $${qparams.length}`; }
|
||||
const { rows } = await pool.query(q, qparams);
|
||||
const files = rows.map(r => {
|
||||
let size = 0;
|
||||
try { size = fs.statSync(path.join(__dirname, r.path)).size; } catch {}
|
||||
const files = await Promise.all(rows.map(async r => {
|
||||
const size = await storage.sizeOf(r.path);
|
||||
return { id: r.id, token: r.token, name: r.name, created_at: r.created_at, size };
|
||||
});
|
||||
}));
|
||||
res.json({ files, total });
|
||||
});
|
||||
|
||||
@@ -3783,14 +3797,17 @@ app.get('/api/files/:token', fileLimiter, async (req, res) => {
|
||||
const { rows } = await pool.query('SELECT path, name FROM project_files WHERE token = $1', [req.params.token]);
|
||||
if (!rows.length) return res.status(404).json({ error: 'Not found' });
|
||||
const r = rows[0];
|
||||
const fp = path.join(__dirname, r.path);
|
||||
if (!fs.existsSync(fp)) return res.status(404).json({ error: 'File missing' });
|
||||
const key = storage.keyFromPath(r.path);
|
||||
if (!key) return res.status(404).json({ error: 'File missing' });
|
||||
if (isImageName(r.name)) {
|
||||
if (req.query.thumb) return sendImageThumb(res, fp);
|
||||
res.setHeader('Cache-Control', 'public, max-age=31536000, immutable');
|
||||
return res.sendFile(fp);
|
||||
if (req.query.thumb) return sendImageThumb(res, key);
|
||||
const ok = await storage.streamTo(res, key, { cacheControl: 'public, max-age=31536000, immutable' });
|
||||
if (!ok && !res.headersSent) return res.status(404).json({ error: 'File missing' });
|
||||
return;
|
||||
}
|
||||
if (!(await storage.streamTo(res, key, { download: true, name: r.name })) && !res.headersSent) {
|
||||
return res.status(404).json({ error: 'File missing' });
|
||||
}
|
||||
return res.download(fp, r.name);
|
||||
});
|
||||
|
||||
app.get('/api/stats', requireAuth, async (req, res) => {
|
||||
@@ -3871,36 +3888,23 @@ app.get('/api/system-info', requireAdmin, async (_, res) => {
|
||||
WHERE photo_path IS NOT NULL AND deleted_at IS NULL
|
||||
`);
|
||||
|
||||
function sumPhotoSizes(paths) {
|
||||
async function sumPhotoSizes(paths) {
|
||||
let bytes = 0;
|
||||
for (const p of paths) {
|
||||
if (!p) continue;
|
||||
const fp = path.join(__dirname, p);
|
||||
try {
|
||||
const st = fs.statSync(fp);
|
||||
if (st.isFile()) bytes += st.size;
|
||||
} catch {}
|
||||
bytes += await storage.sizeOf(p);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
const groupPhotoSizes = await pool.query('SELECT photo_path FROM group_photos');
|
||||
const entryPhotoSizes = await pool.query('SELECT photo_path FROM entries WHERE photo_path IS NOT NULL AND deleted_at IS NULL');
|
||||
const groupPhotoBytes = sumPhotoSizes(groupPhotoSizes.rows.map(r => r.photo_path));
|
||||
const entryPhotoBytes = sumPhotoSizes(entryPhotoSizes.rows.map(r => r.photo_path));
|
||||
const groupPhotoBytes = await sumPhotoSizes(groupPhotoSizes.rows.map(r => r.photo_path));
|
||||
const entryPhotoBytes = await sumPhotoSizes(entryPhotoSizes.rows.map(r => r.photo_path));
|
||||
|
||||
const uploadsDir = path.join(__dirname, 'uploads');
|
||||
let uploadsSize = 0;
|
||||
let uploadsCount = 0;
|
||||
if (fs.existsSync(uploadsDir)) {
|
||||
for (const f of fs.readdirSync(uploadsDir)) {
|
||||
const fp = path.join(uploadsDir, f);
|
||||
if (fs.statSync(fp).isFile()) {
|
||||
uploadsCount++;
|
||||
uploadsSize += fs.statSync(fp).size;
|
||||
}
|
||||
}
|
||||
}
|
||||
const usage = await storage.usage();
|
||||
const uploadsSize = usage.size_bytes;
|
||||
const uploadsCount = usage.count;
|
||||
|
||||
const diskInfo = getDiskInfo();
|
||||
|
||||
@@ -3929,6 +3933,14 @@ app.get('/api/system-info', requireAdmin, async (_, res) => {
|
||||
size: formatBytes(uploadsSize),
|
||||
size_bytes: uploadsSize,
|
||||
},
|
||||
storage: {
|
||||
driver: storage.isRemote() ? 's3' : 'local',
|
||||
bucket: usage.bucket,
|
||||
prefix: usage.prefix,
|
||||
count: usage.count,
|
||||
size: formatBytes(usage.size_bytes),
|
||||
size_bytes: usage.size_bytes,
|
||||
},
|
||||
disk: diskInfo,
|
||||
};
|
||||
});
|
||||
@@ -4285,26 +4297,26 @@ async function swapEntryPhotoFiles(req, entryId, oldPath, newPath, { keepOrigina
|
||||
let originalPath = null;
|
||||
const { rows: prevRows } = await pool.query('SELECT photo_original_path FROM entries WHERE id = $1', [entryId]);
|
||||
const prevOriginal = prevRows.length ? prevRows[0].photo_original_path : null;
|
||||
const oldAbs = path.join(UPLOADS_DIR, String(oldPath).replace(/^\/+/, '').replace(/^uploads\//, ''));
|
||||
const oldKey = storage.keyFromPath(oldPath);
|
||||
if (prevOriginal) {
|
||||
originalPath = prevOriginal;
|
||||
try { if (fs.existsSync(oldAbs)) fs.unlinkSync(oldAbs); } catch {}
|
||||
} else if (keepOriginal && fs.existsSync(oldAbs)) {
|
||||
if (oldKey) safeUnlink(oldKey);
|
||||
} else if (keepOriginal && oldKey && (await storage.exists(oldKey))) {
|
||||
try {
|
||||
const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg');
|
||||
const backupPath = path.join(ORIGINALS_DIR, backupName);
|
||||
fs.renameSync(oldAbs, backupPath);
|
||||
originalPath = `/uploads/.originals/${backupName}`;
|
||||
const backupKey = `.originals/${crypto.randomBytes(12).toString('hex')}${path.extname(oldKey) || '.jpg'}`;
|
||||
if (await storage.copyObject(oldKey, backupKey)) {
|
||||
originalPath = `/uploads/${backupKey}`;
|
||||
safeUnlink(oldKey);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('photo original backup failed:', e);
|
||||
}
|
||||
} else {
|
||||
try { if (fs.existsSync(oldAbs)) fs.unlinkSync(oldAbs); } catch {}
|
||||
} else if (oldKey) {
|
||||
safeUnlink(oldKey);
|
||||
}
|
||||
await pool.query('UPDATE entries SET photo_path = $1, photo_original_path = $2 WHERE id = $3', [newPath, originalPath, entryId]);
|
||||
await pool.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [newPath, entryId, oldPath]);
|
||||
const oldThumb = path.join('uploads', '.thumbs', path.basename(oldPath).replace(/\.[^.]+$/, '') + '.webp');
|
||||
safeUnlink(oldThumb);
|
||||
thumbUnlinkFor(oldPath);
|
||||
await pool.query(
|
||||
'INSERT INTO photo_jobs (entry_id, action, status, before_path, after_path, finished_at) VALUES ($1, $2, $3, $4, $5, now())',
|
||||
[entryId, action, 'done', originalPath, newPath]
|
||||
@@ -4343,7 +4355,9 @@ app.put('/api/entries/:id/photo/enhance', requireAuth, (req, res, next) => {
|
||||
const saturate = clampEnhanceParam(p.saturate, 0, 300, 100);
|
||||
const sharpAmt = clampEnhanceParam(p.sharp, 0, 100, 0);
|
||||
const denoise = clampEnhanceParam(p.denoise, 0, 100, 0);
|
||||
const srcPath = path.join(__dirname, oldPath.replace(/^\/+/, ''));
|
||||
const srcKey = storage.keyFromPath(oldPath);
|
||||
const srcPath = srcKey ? await storage.localize(srcKey) : null;
|
||||
if (!srcPath) return res.status(400).json({ error: 'Файл фото не найден' });
|
||||
let pipeline = sharp(srcPath).rotate();
|
||||
if (denoise > 0) pipeline = pipeline.median(denoise > 70 ? 5 : 3);
|
||||
pipeline = pipeline.modulate({ brightness: brightness / 100, saturation: saturate / 100 });
|
||||
@@ -4355,8 +4369,9 @@ app.put('/api/entries/:id/photo/enhance', requireAuth, (req, res, next) => {
|
||||
pipeline = pipeline.sharpen({ sigma: 0.5 + (sharpAmt / 100) * 1.5, m1: 0, m2: 1 + sharpAmt / 50 });
|
||||
}
|
||||
const newName = crypto.randomBytes(12).toString('hex') + '.jpg';
|
||||
const outPath = path.join(__dirname, 'uploads', newName);
|
||||
const outPath = path.join(UPLOADS_DIR, newName);
|
||||
await pipeline.jpeg({ quality: 92, mozjpeg: true }).toFile(outPath);
|
||||
await storage.persist(newName, outPath);
|
||||
newPath = `/uploads/${newName}`;
|
||||
engine = 'sharp';
|
||||
} else {
|
||||
@@ -4436,8 +4451,7 @@ app.post('/api/entries/:id/photo/jobs/:jobId/apply', requireAuth, async (req, re
|
||||
await client.query('ROLLBACK');
|
||||
return res.status(400).json({ error: 'Некорректный путь результата' });
|
||||
}
|
||||
const afterAbs = path.join(UPLOADS_DIR, String(job.after_path).replace(/^\/+/, '').replace(/^uploads\//, ''));
|
||||
if (!fs.existsSync(afterAbs)) {
|
||||
if (!(await storage.exists(job.after_path))) {
|
||||
await client.query('ROLLBACK');
|
||||
return res.status(400).json({ error: 'Файл результата не найден' });
|
||||
}
|
||||
@@ -4451,21 +4465,22 @@ app.post('/api/entries/:id/photo/jobs/:jobId/apply', requireAuth, async (req, re
|
||||
}
|
||||
let originalPath = entryRows[0].photo_original_path;
|
||||
let beforePath = originalPath;
|
||||
const oldAbs = path.join(UPLOADS_DIR, String(oldPath || '').replace(/^\/+/, '').replace(/^uploads\//, ''));
|
||||
if (oldPath && fs.existsSync(oldAbs)) {
|
||||
const oldKey = oldPath ? storage.keyFromPath(oldPath) : null;
|
||||
if (oldKey && (await storage.exists(oldKey))) {
|
||||
try {
|
||||
const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg');
|
||||
fs.renameSync(oldAbs, path.join(ORIGINALS_DIR, backupName));
|
||||
beforePath = `/uploads/.originals/${backupName}`;
|
||||
if (!originalPath) originalPath = beforePath;
|
||||
const backupKey = `.originals/${crypto.randomBytes(12).toString('hex')}${path.extname(oldKey) || '.jpg'}`;
|
||||
if (await storage.copyObject(oldKey, backupKey)) {
|
||||
beforePath = `/uploads/${backupKey}`;
|
||||
if (!originalPath) originalPath = beforePath;
|
||||
safeUnlink(oldKey);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('photo apply backup failed:', e);
|
||||
}
|
||||
}
|
||||
await client.query('UPDATE entries SET photo_path = $1, photo_original_path = $2 WHERE id = $3', [job.after_path, originalPath, req.params.id]);
|
||||
await client.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [job.after_path, req.params.id, oldPath]);
|
||||
const oldThumb = path.join('uploads', '.thumbs', path.basename(oldPath || '').replace(/\.[^.]+$/, '') + '.webp');
|
||||
safeUnlink(oldThumb);
|
||||
thumbUnlinkFor(oldPath);
|
||||
await client.query(`UPDATE photo_jobs SET applied = true, before_path = COALESCE(before_path, $1) WHERE id = $2`, [beforePath, jobId]);
|
||||
await client.query('COMMIT');
|
||||
await logAudit(req, 'entry.photo.apply', { entry_id: req.params.id, job_id: jobId, before_path: beforePath, after_path: job.after_path });
|
||||
@@ -4544,28 +4559,32 @@ app.post('/api/entries/:id/photo/jobs/:jobId/rollback', requireAuth, async (req,
|
||||
}
|
||||
const beforeName = path.basename(job.before_path);
|
||||
if (!/^[A-Za-z0-9._-]+$/.test(beforeName)) return res.status(400).json({ error: 'Некорректный путь' });
|
||||
const beforeAbs = path.join(ORIGINALS_DIR, beforeName);
|
||||
if (!fs.existsSync(beforeAbs)) return res.status(400).json({ error: 'Файл версии не найден' });
|
||||
const beforeKey = `.originals/${beforeName}`;
|
||||
if (!(await storage.exists(beforeKey))) return res.status(400).json({ error: 'Файл версии не найден' });
|
||||
const { rows: cur } = await pool.query('SELECT photo_path FROM entries WHERE id = $1', [req.params.id]);
|
||||
if (!cur.length) return res.status(404).json({ error: 'Запись не найдена' });
|
||||
const oldPath = cur[0].photo_path;
|
||||
if (!oldPath) return res.status(400).json({ error: 'У записи нет фото' });
|
||||
const ext = path.extname(beforeName) || '.jpg';
|
||||
const newPath = `/uploads/${crypto.randomBytes(12).toString('hex')}${ext}`;
|
||||
fs.copyFileSync(beforeAbs, path.join(UPLOADS_DIR, path.basename(newPath)));
|
||||
const oldAbs = path.join(UPLOADS_DIR, String(oldPath).replace(/^\/+/, '').replace(/^uploads\//, ''));
|
||||
const newKey = `${crypto.randomBytes(12).toString('hex')}${ext}`;
|
||||
const newPath = `/uploads/${newKey}`;
|
||||
if (!(await storage.copyObject(beforeKey, newKey))) {
|
||||
return res.status(400).json({ error: 'Файл версии не найден' });
|
||||
}
|
||||
const oldKey = storage.keyFromPath(oldPath);
|
||||
let rollbackBefore = null;
|
||||
if (fs.existsSync(oldAbs)) {
|
||||
if (oldKey && (await storage.exists(oldKey))) {
|
||||
try {
|
||||
const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg');
|
||||
fs.renameSync(oldAbs, path.join(ORIGINALS_DIR, backupName));
|
||||
rollbackBefore = `/uploads/.originals/${backupName}`;
|
||||
const backupKey = `.originals/${crypto.randomBytes(12).toString('hex')}${path.extname(oldKey) || '.jpg'}`;
|
||||
if (await storage.copyObject(oldKey, backupKey)) {
|
||||
rollbackBefore = `/uploads/${backupKey}`;
|
||||
safeUnlink(oldKey);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('photo rollback backup failed:', e);
|
||||
}
|
||||
}
|
||||
const oldThumb = path.join('uploads', '.thumbs', path.basename(oldPath).replace(/\.[^.]+$/, '') + '.webp');
|
||||
safeUnlink(oldThumb);
|
||||
thumbUnlinkFor(oldPath);
|
||||
await pool.query('UPDATE entries SET photo_path = $1 WHERE id = $2', [newPath, req.params.id]);
|
||||
await pool.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [newPath, req.params.id, oldPath]);
|
||||
await pool.query(
|
||||
@@ -4606,26 +4625,29 @@ app.post('/api/entries/:id/photo/restore-original', requireAuth, async (req, res
|
||||
if (!origPath) return res.status(400).json({ error: 'Оригинал не сохранён' });
|
||||
const origName = path.basename(origPath);
|
||||
if (!/^[A-Za-z0-9._-]+$/.test(origName)) return res.status(400).json({ error: 'Некорректный путь оригинала' });
|
||||
const origAbs = path.join(ORIGINALS_DIR, origName);
|
||||
if (!fs.existsSync(origAbs)) return res.status(400).json({ error: 'Файл оригинала не найден' });
|
||||
const newPath = `/uploads/${crypto.randomBytes(12).toString('hex')}${path.extname(origName) || '.jpg'}`;
|
||||
fs.renameSync(origAbs, path.join(UPLOADS_DIR, path.basename(newPath)));
|
||||
const origKey = `.originals/${origName}`;
|
||||
if (!(await storage.exists(origKey))) return res.status(400).json({ error: 'Файл оригинала не найден' });
|
||||
const newKey = `${crypto.randomBytes(12).toString('hex')}${path.extname(origName) || '.jpg'}`;
|
||||
const newPath = `/uploads/${newKey}`;
|
||||
if (!(await storage.copyObject(origKey, newKey))) return res.status(400).json({ error: 'Файл оригинала не найден' });
|
||||
await storage.del(origKey);
|
||||
const oldPath = rows[0].photo_path;
|
||||
await pool.query('UPDATE entries SET photo_path = $1, photo_original_path = NULL WHERE id = $2', [newPath, req.params.id]);
|
||||
await pool.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [newPath, req.params.id, oldPath]);
|
||||
const oldAbs = path.join(UPLOADS_DIR, String(oldPath).replace(/^\/+/, '').replace(/^uploads\//, ''));
|
||||
const oldKey = storage.keyFromPath(oldPath);
|
||||
let beforePath = null;
|
||||
if (fs.existsSync(oldAbs)) {
|
||||
if (oldKey && (await storage.exists(oldKey))) {
|
||||
try {
|
||||
const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg');
|
||||
fs.renameSync(oldAbs, path.join(ORIGINALS_DIR, backupName));
|
||||
beforePath = `/uploads/.originals/${backupName}`;
|
||||
const backupKey = `.originals/${crypto.randomBytes(12).toString('hex')}${path.extname(oldKey) || '.jpg'}`;
|
||||
if (await storage.copyObject(oldKey, backupKey)) {
|
||||
beforePath = `/uploads/${backupKey}`;
|
||||
safeUnlink(oldKey);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('photo original backup failed:', e);
|
||||
}
|
||||
}
|
||||
const oldThumb = path.join('uploads', '.thumbs', path.basename(oldPath).replace(/\.[^.]+$/, '') + '.webp');
|
||||
safeUnlink(oldThumb);
|
||||
thumbUnlinkFor(oldPath);
|
||||
await pool.query(
|
||||
'INSERT INTO photo_jobs (entry_id, action, status, before_path, after_path, error, finished_at) VALUES ($1, $2, $3, $4, $5, NULL, now())',
|
||||
[req.params.id, 'restore', 'done', beforePath, newPath]
|
||||
@@ -5250,8 +5272,19 @@ if (fs.existsSync(certPath) && fs.existsSync(keyPath)) {
|
||||
try { await pool.query(`INSERT INTO settings (key, value) VALUES ('camera_enabled', 'true') ON CONFLICT (key) DO NOTHING`); } catch (err) { console.error('Camera setting:', err); }
|
||||
try { await pool.query(`INSERT INTO settings (key, value) VALUES ('trash_purge_days', '30') ON CONFLICT (key) DO NOTHING`); } catch (err) { console.error('Trash purge days setting:', err); }
|
||||
try { await sweepOrphanedUploads(); } catch (err) { console.error('Upload sweep:', err); }
|
||||
if (storage.isRemote()) {
|
||||
try {
|
||||
await storage.ensureBucket();
|
||||
console.log('S3 bucket ready:', storage.bucket);
|
||||
} catch (err) {
|
||||
console.error('S3 bucket init:', err.message);
|
||||
}
|
||||
}
|
||||
try { await purgeScheduledDeletions(); } catch (err) { console.error('Trash purge:', err); }
|
||||
setInterval(() => { purgeScheduledDeletions().catch(err => console.error('Trash purge:', err)); }, 60 * 60 * 1000).unref();
|
||||
setInterval(() => {
|
||||
try { storage.pruneCache(); } catch (err) { console.error('Cache prune:', err); }
|
||||
}, 60 * 60 * 1000).unref();
|
||||
entryAutoChecker = createEntryAutoChecker({ pool, getSetting, logAudit, aiUrl: AI_URL, defaultPrompt: AI_DEFAULT_PROMPT });
|
||||
entryAutoChecker.start();
|
||||
console.log('AI auto-check worker started');
|
||||
@@ -5264,6 +5297,7 @@ if (fs.existsSync(certPath) && fs.existsSync(keyPath)) {
|
||||
photoAiUrl: PHOTO_AI_URL,
|
||||
uploadsDir: UPLOADS_DIR,
|
||||
originalsDir: ORIGINALS_DIR,
|
||||
storage,
|
||||
});
|
||||
photoWorker.start();
|
||||
console.log('Photo enhance worker started');
|
||||
|
||||
Reference in New Issue
Block a user