feat(storage): S3-совместимое хранилище файлов (SeaweedFS/MinIO) и миграция uploads

- storage.js: абстракция хранилища с драйверами local и s3 (AWS SDK v3),
  ключи объектов совпадают с текущими путями /uploads/<файл>, поэтому схема БД
  и URL не меняются
- docker-compose.yml: сервис s3 (SeaweedFS, том s3-data, API только на loopback),
  переменные S3_*/STORAGE_*, restart unless-stopped для app и db
- docker-compose.minio.yml: оверрайд S3-сервиса на MinIO (образ из своего зеркала)
- server.js/worker.js: чтение и запись файлов только через storage (отдача
  /uploads/*, миниатюры, share-файлы, zip-отчёты, enhance/apply/rollback,
  photo-worker), автосоздание бакета, глобальная персистенция загрузок multer
- бэкап/восстановление и scripts/backup.sh, restore.sh — через scripts/storage-sync.js
- scripts/migrate-to-s3.js: идемпотентная миграция uploads/ в бакет
  (--dry-run, --verify-only, --delete-local)
- админка: блок «Хранилище» в системной информации
- .env.example, README.md, AGENTS.md: описание драйверов, переменных и перехода на S3
This commit is contained in:
dev
2026-09-26 11:08:53 +03:00
parent 8bf54fb95d
commit ddd49707ae
16 changed files with 1563 additions and 217 deletions
+26
View File
@@ -31,3 +31,29 @@ WG_HANDSHAKE_TIMEOUT=60
# ИИ-улучшение фото (Real-ESRGAN), пусто = контейнер photo-ai # ИИ-улучшение фото (Real-ESRGAN), пусто = контейнер photo-ai
PHOTO_AI_URL= PHOTO_AI_URL=
PHOTO_AI_MAX_PIXELS=4000000 PHOTO_AI_MAX_PIXELS=4000000
# === Хранилище файлов (S3: SeaweedFS по умолчанию / MinIO) ===
# local — файлы в ./uploads (по умолчанию), s3 — объекты в бакете S3/MinIO.
STORAGE_DRIVER=local
# Читать локальную копию, если объект ещё не перенесён в S3 (1 — включено).
STORAGE_LOCAL_FALLBACK=1
# Оставлять локальную копию после выгрузки в S3 (1 — оставлять, 0 — удалять).
STORAGE_KEEP_LOCAL=0
# Сколько часов хранить локальный кэш оригиналов (для sharp/миниатюр), 0 — не чистить.
STORAGE_CACHE_MAX_AGE_HOURS=168
# Образ S3-сервиса. По умолчанию SeaweedFS (свободный S3-сервер).
# Для MinIO: S3_IMAGE=minio/minio:<тег> и запуск через docker-compose.minio.yml.
S3_IMAGE=chrislusf/seaweedfs:latest
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=whatido
# Логин/пароль S3 (для SeaweedFS — AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY,
# для MinIO — MINIO_ROOT_USER/MINIO_ROOT_PASSWORD).
S3_ACCESS_KEY=whatido
S3_SECRET_KEY=замените-на-длинный-секрет
# Path-style адресация (1 — включено, нужно для MinIO и SeaweedFS).
S3_FORCE_PATH_STYLE=1
# Необязательный префикс ключей внутри бакета (например, prod).
S3_PREFIX=
# Показывать веб-консоль MinIO (on/off), только для docker-compose.minio.yml
MINIO_BROWSER=off
+45 -9
View File
@@ -8,9 +8,9 @@ This document defines how AI agents should work with the WhatIDo codebase. Follo
**WhatIDo** — Accounting system for an educational center: attendance journal, student project works, group gallery, detached files, and public showcase pages (share links). **WhatIDo** — Accounting system for an educational center: attendance journal, student project works, group gallery, detached files, and public showcase pages (share links).
- **Stack**: Node.js 20 + Express, PostgreSQL 16, Docker Compose, Tailscale (Serve/Funnel) - **Stack**: Node.js 20 + Express, PostgreSQL 16, Docker Compose, S3-совместимое хранилище файлов, Tailscale (Serve/Funnel)
- **Architecture**: Single Express server (`server.js`) + static frontend in `public/` - **Architecture**: Single Express server (`server.js`) + storage abstraction (`storage.js`) + static frontend in `public/`
- **Deployment**: Docker Compose (app + db + tailscale), bind-mounted uploads, named volume for Postgres data - **Deployment**: Docker Compose (app + db + s3 + tailscale), bind-mounted uploads, named volumes for Postgres and S3 data
- **Auth**: Admin-only via `X-Admin-Token` header (value = `ADMIN_PASSWORD` env var). No user sessions. - **Auth**: Admin-only via `X-Admin-Token` header (value = `ADMIN_PASSWORD` env var). No user sessions.
--- ---
@@ -34,9 +34,18 @@ This document defines how AI agents should work with the WhatIDo codebase. Follo
### 3. File Uploads ### 3. File Uploads
- **Multer configs**: `upload` (images only), `adminUpload` (wider allowed ext), `uploadBackup` (restore) - **Multer configs**: `upload` (images only), `adminUpload` (wider allowed ext), `uploadBackup` (restore)
- **Limits**: 10 MB/file, 30 MB total per entry - **Limits**: 10 MB/file, 30 MB total per entry
- **Storage**: `uploads/` bind-mounted to host, filenames = `timestamp-random.ext` - **Staging**: Multer always writes to `uploads/` (`timestamp-random.ext`); a global `res.on('finish')` hook persists each uploaded file through `storage.persist` on successful responses (only when `STORAGE_DRIVER=s3`)
- **HEIC**: Auto-converted to JPEG via `heic-convert` - **HEIC**: Auto-converted to JPEG via `heic-convert`
- **Cleanup**: `safeUnlink` / `sweepOrphanedUploads` — never delete outside `uploads/` - **Cleanup**: `safeUnlink` / `sweepOrphanedUploads` — never delete outside `uploads/` or the configured bucket
### 3a. Storage (`storage.js`)
- **Drivers**: `local` (default, files in `uploads/`) and `s3` (S3-compatible: SeaweedFS by default, MinIO via `docker-compose.minio.yml`)
- **Keys are stable**: DB stores `/uploads/<name>`; S3 object keys are the same `<name>` (plus `.originals/<name>`). Never change key format — it would break existing DB rows and URLs
- **API**: `put`, `putFile`, `head`, `exists`, `sizeOf`, `getStream`, `getBuffer`, `del`, `copyObject`, `listAll`, `localize`, `persist`, `streamTo`, `downloadAll`, `uploadTree`, `ensureBucket`, `usage`, `pruneCache`
- **Rules**: never call `fs.*` on `uploads/` directly in request/worker code — use `storage.*`. `safeUnlink` is the only deletion helper (local + remote, idempotent)
- **Read path**: `STORAGE_LOCAL_FALLBACK=1` prefers a local file when it still exists (covers in-flight uploads and partial migration); otherwise the app streams the object from S3
- **Cache**: `.thumbs` (WebP miniatures) and `.cache` (originals localized for sharp/zip) live inside `uploads/` and are pruned hourly (`STORAGE_CACHE_MAX_AGE_HOURS`)
- **Never publish the S3 API port**: only `127.0.0.1` on the host, file access stays behind app auth/rate limits
### 4. API Patterns ### 4. API Patterns
- **Admin routes**: `requireAdmin` middleware (checks `X-Admin-Token`) - **Admin routes**: `requireAdmin` middleware (checks `X-Admin-Token`)
@@ -102,6 +111,15 @@ This document defines how AI agents should work with the WhatIDo codebase. Follo
- Update Multer `fileFilter` functions - Update Multer `fileFilter` functions
- Keep `MAX_TOTAL_UPLOAD_BYTES` and per-file limit in sync - Keep `MAX_TOTAL_UPLOAD_BYTES` and per-file limit in sync
### Migrate files to S3 / switch storage driver
1. `docker compose up -d s3`
2. `docker compose exec -T app node scripts/migrate-to-s3.js --dry-run` then without the flag (idempotent, size-checked, keeps local files)
3. `docker compose exec -T app node scripts/migrate-to-s3.js --verify-only`
4. Set `STORAGE_DRIVER=s3` in `.env`, `docker compose up -d app`
5. After verification: `docker compose exec -T app node scripts/migrate-to-s3.js --delete-local`
- Rollback: `STORAGE_DRIVER=local` + `docker compose up -d app`
- Do not run `--delete-local` before the app serves reads from S3 and the verification passes
--- ---
## Testing & Verification ## Testing & Verification
@@ -121,6 +139,11 @@ curl -H "X-Admin-Token: $ADMIN_PASSWORD" http://localhost:3003/api/groups
# Run backup/restore scripts # Run backup/restore scripts
./scripts/backup.sh ./scripts/backup.sh
./scripts/restore.sh backups/whatido-backup-<date>.tar.gz ./scripts/restore.sh backups/whatido-backup-<date>.tar.gz
# Storage checks
docker compose up -d s3
docker compose exec -T app node scripts/migrate-to-s3.js --dry-run
docker compose exec -T app node scripts/migrate-to-s3.js --verify-only
``` ```
--- ---
@@ -128,6 +151,7 @@ curl -H "X-Admin-Token: $ADMIN_PASSWORD" http://localhost:3003/api/groups
## Security Checklist (before any change) ## Security Checklist (before any change)
- [ ] No SQL interpolation — only `$1`, `$2`... - [ ] No SQL interpolation — only `$1`, `$2`...
- [ ] Upload path validation via `isSafeUploadPath` / `safeUnlink` - [ ] Upload path validation via `isSafeUploadPath` / `safeUnlink`
- [ ] File I/O через `storage.*`, ключи объектов не выходят за пределы бакета/`uploads/`
- [ ] Rate limiter on new public routes - [ ] Rate limiter on new public routes
- [ ] Admin routes behind `requireAdmin` - [ ] Admin routes behind `requireAdmin`
- [ ] No secrets in code — only via env vars - [ ] No secrets in code — only via env vars
@@ -141,15 +165,19 @@ curl -H "X-Admin-Token: $ADMIN_PASSWORD" http://localhost:3003/api/groups
| File | Purpose | | File | Purpose |
|------|---------| |------|---------|
| `server.js` | Entire backend (Express, routes, DB, uploads, backup) | | `server.js` | Entire backend (Express, routes, DB, uploads, backup) |
| `worker.js` | Background AI auto-check worker for entry messages | | `storage.js` | Storage abstraction: `local` and `s3` drivers, key normalization, cache/thumb helpers |
| `worker.js` | Background AI auto-check worker for entry messages + photo enhance worker |
| `db/init.sql` | Initial schema (runs on fresh DB) | | `db/init.sql` | Initial schema (runs on fresh DB) |
| `db/migration.sql` | Idempotent migrations for existing DBs | | `db/migration.sql` | Idempotent migrations for existing DBs |
| `docker-compose.yml` | Service definitions (app, db, tailscale) | | `docker-compose.yml` | Service definitions (app, db, s3, tailscale) |
| `docker-compose.minio.yml` | Override: S3 service backed by MinIO instead of SeaweedFS |
| `Dockerfile` | App image build | | `Dockerfile` | App image build |
| `public/*.html` | Frontend pages | | `public/*.html` | Frontend pages |
| `public/admin.js` | Shared frontend logic | | `public/admin.js` | Shared frontend logic |
| `scripts/backup.sh` | Host-level backup script | | `scripts/backup.sh` | Host-level backup script (DB dump + storage export) |
| `scripts/restore.sh` | Host-level restore script | | `scripts/restore.sh` | Host-level restore script (DB dump + storage import) |
| `scripts/storage-sync.js` | Export/import all storage objects (used by backup/restore) |
| `scripts/migrate-to-s3.js` | One-off/idempotent migration `uploads/` -> S3 bucket |
| `scripts/deploy.sh` | Deploy script (pull master, build image with commit version, restart app) | | `scripts/deploy.sh` | Deploy script (pull master, build image with commit version, restart app) |
| `start-tailscale.sh` | Tailscale container entrypoint | | `start-tailscale.sh` | Tailscale container entrypoint |
| `.env.example` | Env var template | | `.env.example` | Env var template |
@@ -160,6 +188,9 @@ curl -H "X-Admin-Token: $ADMIN_PASSWORD" http://localhost:3003/api/groups
- ❌ Add dependencies without updating `package.json` and rebuilding - ❌ Add dependencies without updating `package.json` and rebuilding
- ❌ Write files outside `uploads/` or `certs/` - ❌ Write files outside `uploads/` or `certs/`
- ❌ Touch `uploads/` with `fs.*` in request/worker code — use `storage.*` (files may live only in S3)
- ❌ Run `migrate-to-s3.js --delete-local` before verification and cutover
- ❌ Expose the S3 API port publicly (only `127.0.0.1` in compose)
- ❌ Commit `.env`, `certs/`, `uploads/`, `backups/`, `node_modules/` - ❌ Commit `.env`, `certs/`, `uploads/`, `backups/`, `node_modules/`
- ❌ Expose DB port (5432) outside docker network - ❌ Expose DB port (5432) outside docker network
- ❌ Use `eval`, `Function` constructor, or dynamic code execution - ❌ Use `eval`, `Function` constructor, or dynamic code execution
@@ -182,6 +213,11 @@ docker compose logs -f app
# DB shell # DB shell
docker compose exec db psql -U app -d whereldo docker compose exec db psql -U app -d whereldo
# S3 storage status and migration verification
docker compose up -d s3
docker compose exec -T app node scripts/migrate-to-s3.js --verify-only
docker compose exec -T app node scripts/migrate-to-s3.js --delete-local
# Tailscale status # Tailscale status
docker exec -it whatido-tailscale-1 tailscale status docker exec -it whatido-tailscale-1 tailscale status
+75 -5
View File
@@ -12,6 +12,7 @@
- **Share-ссылки** — публичные страницы-витрины с выбором группы / воспитанника / диапазона дат - **Share-ссылки** — публичные страницы-витрины с выбором группы / воспитанника / диапазона дат
- **Дашборд** — статистика, активные группы, активность за 14 дней, последние записи, топ воспитанников - **Дашборд** — статистика, активные группы, активность за 14 дней, последние записи, топ воспитанников
- **Резервное копирование** — экспорт/импорт полного дампа (БД + файлы) в `tar.gz` - **Резервное копирование** — экспорт/импорт полного дампа (БД + файлы) в `tar.gz`
- **Хранилище файлов** — локальный каталог `uploads/` или S3-совместимый сервис (`s3`: SeaweedFS, либо MinIO через оверрайд), перенос файлов скриптом миграции
- **Настройки** — тексты футера, анти-спам интервал - **Настройки** — тексты футера, анти-спам интервал
- **Публикация через Tailscale** — приложение открывается по постоянному адресу `https://whatido.<tailnet>.ts.net` без проброса портов, внешнего IP и reverse-proxy - **Публикация через Tailscale** — приложение открывается по постоянному адресу `https://whatido.<tailnet>.ts.net` без проброса портов, внешнего IP и reverse-proxy
@@ -20,6 +21,7 @@
- Node.js + Express - Node.js + Express
- PostgreSQL (pg) - PostgreSQL (pg)
- Multer (загрузка файлов), Tar (бэкапы) - Multer (загрузка файлов), Tar (бэкапы)
- S3-совместимое хранилище (AWS SDK v3): сервис `s3` (SeaweedFS / MinIO)
- Lucide (иконки UI) - Lucide (иконки UI)
- Docker / Docker Compose - Docker / Docker Compose
- Tailscale (Serve / Funnel) — публикация по HTTPS - Tailscale (Serve / Funnel) — публикация по HTTPS
@@ -244,7 +246,71 @@ docker compose exec cloudflared wg show # есть handshake — VPN подн
## Хранилище файлов ## Хранилище файлов
Загруженные фото и файлы хранятся в каталоге `uploads/` на хосте и монтируются в контейнер (`./uploads:/app/uploads`). Это даёт прямой доступ к данным из-под хост-системы. Данные БД хранятся в именованном томе `pgdata`. По умолчанию загруженные фото и файлы хранятся в каталоге `uploads/` на хосте и монтируются в контейнер (`./uploads:/app/uploads`) — это драйвер `local`. Данные БД хранятся в именованном томе `pgdata`.
Дополнительно поддерживается **S3-совместимое хранилище** (сервис `s3` в compose, драйвер `s3`). Все обращения к файлам идут через приложение: URL (`/uploads/...`, `/uploads/thumb/...`, `/api/files/:token`, share-ссылки) и записи в БД (`/uploads/<файл>`) не меняются, поэтому переключение драйвера не требует миграции данных в БД.
### Сервис `s3`
```bash
docker compose up -d s3 # поднимает S3-хранилище (том s3-data)
```
- **По умолчанию — SeaweedFS** (`chrislusf/seaweedfs`): свободный S3-сервер; API слушает `127.0.0.1:9000` на хосте и `s3:9000` внутри compose-сети.
- **MinIO**: официальные свободные образы `minio/minio` удалены из Docker Hub, поэтому MinIO подключается через оверрайд и образ из доступного вам зеркала:
```bash
S3_IMAGE=<ваш-образ-minio> docker compose -f docker-compose.yml -f docker-compose.minio.yml up -d s3
```
Бакет создаётся автоматически при старте приложения (`ensureBucket`) или скриптом миграции. Анонимный доступ к API хранилища закрыт: порт `9000` не публикуется наружу (только loopback), доступ к файлам остаётся через приложение с его аутентификацией и rate limit.
### Переменные окружения
| Переменная | По умолчанию | Назначение |
|---|---|---|
| `STORAGE_DRIVER` | `local` | `local` — файлы в `uploads/`, `s3` — объекты в бакете |
| `S3_ENDPOINT` | `http://s3:9000` | Адрес S3 API внутри compose-сети |
| `S3_BUCKET` | `whatido` | Бакет для объектов |
| `S3_ACCESS_KEY` / `S3_SECRET_KEY` | `whatido` / — | Доступ к хранилищу (для MinIO это root-пользователь) |
| `S3_FORCE_PATH_STYLE` | `1` | Path-style адресация (нужна MinIO/SeaweedFS) |
| `S3_PREFIX` | — | Необязательный префикс ключей внутри бакета |
| `STORAGE_LOCAL_FALLBACK` | `1` | Читать локальный файл, если объекта в S3 ещё нет |
| `STORAGE_KEEP_LOCAL` | `0` | Оставлять локальную копию после выгрузки в S3 |
| `STORAGE_CACHE_MAX_AGE_HOURS` | `168` | Срок жизни локального кэша оригиналов (для sharp/миниатюр) |
### Переход на S3 (миграция)
Порядок не прерывает работу: файлы сначала копируются в бакет, локальные остаются на месте и продолжают использоваться.
```bash
# 1) поднять хранилище
docker compose up -d s3
# 2) предпросмотр и загрузка файлов в бакет (идемпотентно, по размеру объекта)
docker compose exec -T app node scripts/migrate-to-s3.js --dry-run
docker compose exec -T app node scripts/migrate-to-s3.js
# 3) проверить, что все объекты на месте (ничего не меняет)
docker compose exec -T app node scripts/migrate-to-s3.js --verify-only
```
Дальше включить драйвер `s3` и перезапустить приложение:
```bash
# в .env: STORAGE_DRIVER=s3
docker compose up -d app
```
Новые загрузки уходят в бакет (локальная копия удаляется, если `STORAGE_KEEP_LOCAL=0`), старые файлы ещё читаются из `uploads/` благодаря `STORAGE_LOCAL_FALLBACK=1`. Когда всё проверено — удалите локальные копии:
```bash
docker compose exec -T app node scripts/migrate-to-s3.js --delete-local
```
Откат в любой момент: `STORAGE_DRIVER=local` + `docker compose up -d app` (пока локальные копии не удалены).
Объём и состав хранилища видны в админке: Настройки → Системная информация (блок «Хранилище»).
## Бэкапы ## Бэкапы
@@ -264,6 +330,8 @@ docker compose exec cloudflared wg show # есть handshake — VPN подн
Форматы не взаимозаменяемы: скриптовый архив содержит `db.sql.gz` + `_uploads/` (перенос на другой хост через `scripts/restore.sh`), а веб-архив из админки — `data.json` + `uploads/` (кнопка «Восстановить»). Если в админку загрузить скриптовый архив, сервер вернёт подсказку, какой инструмент использовать. Форматы не взаимозаменяемы: скриптовый архив содержит `db.sql.gz` + `_uploads/` (перенос на другой хост через `scripts/restore.sh`), а веб-архив из админки — `data.json` + `uploads/` (кнопка «Восстановить»). Если в админку загрузить скриптовый архив, сервер вернёт подсказку, какой инструмент использовать.
Файлы попадают в бэкап из активного хранилища: при `STORAGE_DRIVER=s3` админ-бэкап и `scripts/backup.sh` выгружают объекты из бакета (`scripts/storage-sync.js export`), а восстановление загружает их обратно (`scripts/storage-sync.js import`). Миниатюры (`.thumbs`) в архив не включаются — они пересоздаются по запросу.
## Безопасность ## Безопасность
- **Пароль администратора** обязателен (`ADMIN_PASSWORD`); фолбэка на `admin` нет. - **Пароль администратора** обязателен (`ADMIN_PASSWORD`); фолбэка на `admin` нет.
@@ -299,17 +367,19 @@ docker compose exec cloudflared wg show # есть handshake — VPN подн
## Структура проекта ## Структура проекта
``` ```
├── docker-compose.yml # сервисы: app + db + tailscale ├── docker-compose.yml # сервисы: app + db + s3 (+ опционально tailscale)
├── docker-compose.minio.yml # оверрайд: S3-сервис на MinIO вместо SeaweedFS
├── .env.example # шаблон переменных окружения ├── .env.example # шаблон переменных окружения
├── Dockerfile # сборка образа (Node 20, генерация TLS-сертификата) ├── Dockerfile # сборка образа (Node 20, генерация TLS-сертификата)
├── server.js # Express-приложение ├── server.js # Express-приложение
├── worker.js # фоновый worker AI-проверки записей ├── storage.js # абстракция хранилища: драйверы local и s3
├── worker.js # фоновый worker AI-проверки и ИИ-улучшения фото
├── certs/ # cert.pem приложения (монтируется в tailscale, в git не хранится) ├── certs/ # cert.pem приложения (монтируется в tailscale, в git не хранится)
├── db/ ├── db/
│ ├── init.sql # схема при первом запуске │ ├── init.sql # схема при первом запуске
│ └── migration.sql # миграции существующей БД │ └── migration.sql # миграции существующей БД
├── public/ # статика (HTML/CSS/JS админки и витрин) ├── public/ # статика (HTML/CSS/JS админки и витрин)
├── scripts/ # вспомогательные скрипты ├── scripts/ # вспомогательные скрипты (backup/restore/deploy, migrate-to-s3, storage-sync)
├── uploads/ # загруженные файлы (bind-монт, вне git) ├── uploads/ # локальные файлы и кэш миниатюр (bind-монт, вне git)
└── backups/ # локальные бэкапы └── backups/ # локальные бэкапы
``` ```
+26
View File
@@ -0,0 +1,26 @@
# Переопределение S3-сервиса на MinIO.
# Использование:
# S3_IMAGE=minio/minio:RELEASE.2025-04-22T22-12-26Z \
# docker compose -f docker-compose.yml -f docker-compose.minio.yml up -d s3
#
# Учтите: MinIO прекратил публикацию свободных образов (docker.io/minio/minio
# удалён), поэтому образ нужно взять из доступного вам зеркала/архива и указать
# его в S3_IMAGE.
services:
s3:
image: ${S3_IMAGE:-minio/minio:latest}
container_name: whatido-s3
restart: unless-stopped
command: server /data --console-address ":9001"
environment:
MINIO_ROOT_USER: ${S3_ACCESS_KEY:-whatido}
MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY:-whatido-secret}
MINIO_BROWSER: ${MINIO_BROWSER:-off}
TZ: Europe/Moscow
expose:
- "9000"
ports:
- "127.0.0.1:9000:9000"
- "127.0.0.1:9001:9001"
volumes:
- s3-data:/data
+37
View File
@@ -3,6 +3,7 @@ name: whatido
services: services:
db: db:
image: postgres:16-alpine image: postgres:16-alpine
restart: unless-stopped
environment: environment:
POSTGRES_DB: whereldo POSTGRES_DB: whereldo
POSTGRES_USER: app POSTGRES_USER: app
@@ -24,6 +25,7 @@ services:
args: args:
GIT_COMMIT: ${GIT_COMMIT:-} GIT_COMMIT: ${GIT_COMMIT:-}
GIT_COMMIT_DATE: ${GIT_COMMIT_DATE:-} GIT_COMMIT_DATE: ${GIT_COMMIT_DATE:-}
restart: unless-stopped
extra_hosts: extra_hosts:
- "host.docker.internal:host-gateway" - "host.docker.internal:host-gateway"
expose: expose:
@@ -43,12 +45,46 @@ services:
PHOTO_AI_URL: ${PHOTO_AI_URL:-http://photo-ai:8080} PHOTO_AI_URL: ${PHOTO_AI_URL:-http://photo-ai:8080}
NODE_ENV: production NODE_ENV: production
TZ: Europe/Moscow TZ: Europe/Moscow
STORAGE_DRIVER: ${STORAGE_DRIVER:-local}
STORAGE_LOCAL_FALLBACK: ${STORAGE_LOCAL_FALLBACK:-1}
STORAGE_KEEP_LOCAL: ${STORAGE_KEEP_LOCAL:-0}
STORAGE_CACHE_MAX_AGE_HOURS: ${STORAGE_CACHE_MAX_AGE_HOURS:-168}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-whatido}
S3_ACCESS_KEY: ${S3_ACCESS_KEY:-whatido}
S3_SECRET_KEY: ${S3_SECRET_KEY:-whatido-secret}
S3_FORCE_PATH_STYLE: ${S3_FORCE_PATH_STYLE:-1}
S3_PREFIX: ${S3_PREFIX:-}
depends_on: depends_on:
db: db:
condition: service_healthy condition: service_healthy
volumes: volumes:
- ./uploads:/app/uploads - ./uploads:/app/uploads
# S3-совместимое хранилище файлов. API 9000 доступен только внутри сети compose
# (плюс loopback хоста для отладки/миграции).
# По умолчанию — SeaweedFS: свободный S3-сервер, доступный в Docker Hub.
# Для MinIO (если образ доступен в вашем зеркале) используйте:
# docker compose -f docker-compose.yml -f docker-compose.minio.yml up -d s3
# Перенос файлов из ./uploads в бакет:
# docker compose exec -T app node scripts/migrate-to-s3.js --dry-run
s3:
image: ${S3_IMAGE:-chrislusf/seaweedfs:latest}
container_name: whatido-s3
restart: unless-stopped
command: server -dir=/data -s3 -s3.port=9000
environment:
AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-whatido}
AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY:-whatido-secret}
TZ: Europe/Moscow
expose:
- "9000"
ports:
- "127.0.0.1:9000:9000"
volumes:
- s3-data:/data
# Публикация через Tailscale (Serve / Funnel) без проброса портов. # Публикация через Tailscale (Serve / Funnel) без проброса портов.
# Приложение доступно по https://whatido.<tailnet>.ts.net # Приложение доступно по https://whatido.<tailnet>.ts.net
# tailscale: # tailscale:
@@ -134,3 +170,4 @@ services:
volumes: volumes:
pgdata: pgdata:
photo-ai-models: photo-ai-models:
s3-data:
+397 -2
View File
@@ -8,6 +8,7 @@
"name": "whereldo", "name": "whereldo",
"version": "1.0.0", "version": "1.0.0",
"dependencies": { "dependencies": {
"@aws-sdk/client-s3": "^3.1141.0",
"bcrypt": "^5.1.1", "bcrypt": "^5.1.1",
"express": "^4.21.0", "express": "^4.21.0",
"express-rate-limit": "^8.7.0", "express-rate-limit": "^8.7.0",
@@ -20,6 +21,314 @@
"tar": "^7.4.3" "tar": "^7.4.3"
} }
}, },
"node_modules/@aws-sdk/checksums": {
"version": "3.1001.1",
"resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1001.1.tgz",
"integrity": "sha512-x12Q17KYlJAd3nKf8LV5LV0vt8sh8/6YfQLGPtrGnQf/tW4jqxPGq5GPpuVitpQYM3eUR4XB7CbxZf751NMbLw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/client-s3": {
"version": "3.1141.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1141.0.tgz",
"integrity": "sha512-uOVH37xGLenAdJkCPCin/JJG2PgWrFcSsDnQ9+C9Zq8N9Oalo5ol4xmn5fG28iWAlA/b/9boQZgHbMh+UsIhcg==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/checksums": "^3.1001.1",
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/credential-provider-node": "^3.972.84",
"@aws-sdk/middleware-sdk-s3": "^3.972.77",
"@aws-sdk/signature-v4-multi-region": "^3.996.47",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/fetch-http-handler": "^5.8.0",
"@smithy/node-http-handler": "^4.12.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/core": {
"version": "3.978.1",
"resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.978.1.tgz",
"integrity": "sha512-LbY9aGsEiznDWmUc30Nwv3aIX/+dbwTx8KfS0yOC3NPYMO+O91e6jkT1azf34FwjOndq8/Q+RcVVZz5xnerwdg==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/types": "^3.974.6",
"@aws-sdk/xml-builder": "^3.972.41",
"@aws/lambda-invoke-store": "^0.3.0",
"@smithy/core": "^3.35.0",
"@smithy/signature-v4": "^5.7.3",
"@smithy/types": "^4.19.0",
"bowser": "^2.11.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-env": {
"version": "3.972.72",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.72.tgz",
"integrity": "sha512-xTKO/FWJPozTIXbozVnVGoNBhaGba8TBcx+KyUjRVeOlXE+dUc7GTR1cLvu0uTdIdmemzaFbqqCshXeZA1fZew==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-http": {
"version": "3.972.74",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.74.tgz",
"integrity": "sha512-u91E/hT8f4d1xy0Jl7VG4nVKJ3lxbrZkoBTeSVoJdWBiSEUMwMS/9+e0H/aJVQV//Lt5wuzP+E69v4aRSsNTmw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/fetch-http-handler": "^5.8.0",
"@smithy/node-http-handler": "^4.12.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-ini": {
"version": "3.973.17",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.17.tgz",
"integrity": "sha512-ged4KXdBkvIC81bLvNHHuQKdKak/VXhQTR1NWYTTqW0474nlmsxy9O/vlgTIohDDWH3xpBdtVMZRyjb+DnocDA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/credential-provider-env": "^3.972.72",
"@aws-sdk/credential-provider-http": "^3.972.74",
"@aws-sdk/credential-provider-login": "^3.972.79",
"@aws-sdk/credential-provider-process": "^3.972.72",
"@aws-sdk/credential-provider-sso": "^3.973.16",
"@aws-sdk/credential-provider-web-identity": "^3.972.78",
"@aws-sdk/nested-clients": "^3.997.46",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/credential-provider-imds": "^4.5.2",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-login": {
"version": "3.972.79",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.79.tgz",
"integrity": "sha512-L+Z85anONJd8MaiuraO4wRxATCdEejBZ3K3eymzWI5JPXa9sOS9CkIm72PBKqXKX+Z9p9NGMX5AIMXm0LEflgw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/nested-clients": "^3.997.46",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-node": {
"version": "3.972.84",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.84.tgz",
"integrity": "sha512-oHt854odINVwzwsh+c5x69j0ajm4DbqqqVJ+O1ECsCIZeMDAbzFpXItaqP7UZstJj/ATdTk/KFSH0LaNAgV+kA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/credential-provider-env": "^3.972.72",
"@aws-sdk/credential-provider-http": "^3.972.74",
"@aws-sdk/credential-provider-ini": "^3.973.17",
"@aws-sdk/credential-provider-process": "^3.972.72",
"@aws-sdk/credential-provider-sso": "^3.973.16",
"@aws-sdk/credential-provider-web-identity": "^3.972.78",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/credential-provider-imds": "^4.5.2",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-process": {
"version": "3.972.72",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.72.tgz",
"integrity": "sha512-rLIp2xbMjX/k9/od7APpqq1ZgXXnV0pOL1Th3ZsL8Wu0TRtBsDTVS8iPqcfRFcHakFxPvR04OSTv2ka2qOb/2A==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-sso": {
"version": "3.973.16",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.16.tgz",
"integrity": "sha512-IGihaJfFZYacJJr/odqILCoK7W/mvrZ7cuK7ECn3sAu4vLC6u0V8bS7mCGbdugJ8Aum2tnvqmx0F2MRFp2rn9g==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/nested-clients": "^3.997.46",
"@aws-sdk/token-providers": "3.1138.0",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/credential-provider-web-identity": {
"version": "3.972.78",
"resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.78.tgz",
"integrity": "sha512-/y9WvNtlcPBGLR0qc1a+9J/xtYZfVczvLUOuXaVWylzttH7ewsxwHtjmiJSolNrVSDorIxHGHMU61CbonRkmwA==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/nested-clients": "^3.997.46",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/middleware-sdk-s3": {
"version": "3.972.77",
"resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.77.tgz",
"integrity": "sha512-E7W2UOeUoc+lg3uIfR/dM7ZwusHwhBQrKMnlkRv4EXRR+C0YtV1pg25xC7GdZIhXH+NAMgZPCbE7o5to2cjFiw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/signature-v4-multi-region": "^3.996.47",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/nested-clients": {
"version": "3.997.46",
"resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.46.tgz",
"integrity": "sha512-oRxtBcka/JGHGs9l9p9IVajGoTP8vTPmoAzdHGy4Qcy9P5vPnDf6nhIeM/COQNY9k/OahImTRaLkHftoXvfcmQ==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/signature-v4-multi-region": "^3.996.47",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/fetch-http-handler": "^5.8.0",
"@smithy/node-http-handler": "^4.12.1",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/signature-v4-multi-region": {
"version": "3.996.47",
"resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.47.tgz",
"integrity": "sha512-Zk08macMvQTHzQJCLJVkOlviVoqwYMrpXv4lmLN7b7sAbiMoOK7Go0NYdR5UeF+MW8LIbRmwrNy9u/5VvX1U5g==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/types": "^3.974.6",
"@smithy/signature-v4": "^5.7.3",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/token-providers": {
"version": "3.1138.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1138.0.tgz",
"integrity": "sha512-GpyAr0DD63YOEmYFM6Df+gJuIgC92MMTiBK4FTKfxii5MJ9ge20epR7LyroulscYlG89J+ZB2ivFDPjvfQhzdw==",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/core": "^3.978.1",
"@aws-sdk/nested-clients": "^3.997.46",
"@aws-sdk/types": "^3.974.6",
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/types": {
"version": "3.974.6",
"resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.6.tgz",
"integrity": "sha512-v/clNZzZnDxGyvpHMOGpJKVXFAExJzUNAAjaWGdcx8QAcXLGwTaOkw33p5SHAi0YAioK32xB3hWwOekRVfmfKg==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws-sdk/xml-builder": {
"version": "3.972.41",
"resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.41.tgz",
"integrity": "sha512-ctjVSyCMegrWfXlx6VqzSBFI6UqmQ5ZlnfMhdLIiWmhoH8UAQxSCP5N3OpG7X3k4LnS7ou74C4mt20+bfTW2aQ==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=20.0.0"
}
},
"node_modules/@aws/lambda-invoke-store": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz",
"integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==",
"license": "Apache-2.0",
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@emnapi/runtime": { "node_modules/@emnapi/runtime": {
"version": "1.11.3", "version": "1.11.3",
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz",
@@ -654,6 +963,87 @@
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
"license": "ISC" "license": "ISC"
}, },
"node_modules/@smithy/core": {
"version": "3.35.0",
"resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.35.0.tgz",
"integrity": "sha512-zRMhfkByhT2snNdr1si24vJitU6Cr9ix2MikUfWmkAgp4jrNP0GcKSP5YvwQ+TlI8AZXER5QOGJn3JsVtSD9/A==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/credential-provider-imds": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz",
"integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.33.2",
"@smithy/types": "^4.17.2",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/fetch-http-handler": {
"version": "5.8.0",
"resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.8.0.tgz",
"integrity": "sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.33.3",
"@smithy/types": "^4.18.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/node-http-handler": {
"version": "4.12.1",
"resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz",
"integrity": "sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.33.3",
"@smithy/types": "^4.18.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/signature-v4": {
"version": "5.7.4",
"resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.4.tgz",
"integrity": "sha512-tHy0K0VtqNd5Y7Y41h0a0Lhh0L1GzC08dTWg0F7vRJWFtTENg7IZikf3wQkanYIRdb7ngoIPMTmqgUi401fEeQ==",
"license": "Apache-2.0",
"dependencies": {
"@smithy/core": "^3.35.0",
"@smithy/types": "^4.19.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@smithy/types": {
"version": "4.19.0",
"resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.19.0.tgz",
"integrity": "sha512-r7jh49VJxGerfAcTQA6gXcKc+98zOp/tqRwzYjgOE+iSQsP6cEU1hq2QzbuipmP68QtYdY9wKEhiCQZIzHgZ4Q==",
"license": "Apache-2.0",
"dependencies": {
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/abbrev": { "node_modules/abbrev": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@@ -807,6 +1197,12 @@
"npm": "1.2.8000 || >= 1.4.16" "npm": "1.2.8000 || >= 1.4.16"
} }
}, },
"node_modules/bowser": {
"version": "2.14.1",
"resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz",
"integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==",
"license": "MIT"
},
"node_modules/brace-expansion": { "node_modules/brace-expansion": {
"version": "1.1.18", "version": "1.1.18",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz",
@@ -2402,8 +2798,7 @@
"version": "2.8.1", "version": "2.8.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
"license": "0BSD", "license": "0BSD"
"optional": true
}, },
"node_modules/type-is": { "node_modules/type-is": {
"version": "1.6.18", "version": "1.6.18",
+1
View File
@@ -6,6 +6,7 @@
"start": "node server.js" "start": "node server.js"
}, },
"dependencies": { "dependencies": {
"@aws-sdk/client-s3": "^3.1141.0",
"bcrypt": "^5.1.1", "bcrypt": "^5.1.1",
"express": "^4.21.0", "express": "^4.21.0",
"express-rate-limit": "^8.7.0", "express-rate-limit": "^8.7.0",
+8
View File
@@ -238,6 +238,7 @@ function renderSystemInfo(si) {
const photos = si.photos || {}; const photos = si.photos || {};
const files = si.files || {}; const files = si.files || {};
const uploads = si.uploads || {}; const uploads = si.uploads || {};
const storage = si.storage || {};
const disk = si.disk; const disk = si.disk;
let tablesHtml = ''; let tablesHtml = '';
@@ -283,6 +284,13 @@ function renderSystemInfo(si) {
<div class="sys-item"><span class="sys-label">Файлов</span><span class="sys-value">${uploads.count || 0}</span></div> <div class="sys-item"><span class="sys-label">Файлов</span><span class="sys-value">${uploads.count || 0}</span></div>
<div class="sys-item"><span class="sys-label">Размер</span><span class="sys-value">${esc(uploads.size || '0 B')}</span></div> <div class="sys-item"><span class="sys-label">Размер</span><span class="sys-value">${esc(uploads.size || '0 B')}</span></div>
</div> </div>
<div class="sys-card">
<div class="sys-title">Хранилище</div>
<div class="sys-item"><span class="sys-label">Драйвер</span><span class="sys-value">${storage.driver === 's3' ? 'S3' : 'Локальное'}</span></div>
${storage.bucket ? `<div class="sys-item"><span class="sys-label">Бакет</span><span class="sys-value">${esc(storage.bucket)}${storage.prefix ? '/' + esc(storage.prefix) : ''}</span></div>` : ''}
<div class="sys-item"><span class="sys-label">Объектов</span><span class="sys-value">${storage.count || 0}</span></div>
<div class="sys-item"><span class="sys-label">Размер</span><span class="sys-value">${esc(storage.size || '0 B')}</span></div>
</div>
${diskHtml || '<div class="sys-card"><div class="sys-title">Диск</div><div class="sys-meta">Недоступно</div></div>'} ${diskHtml || '<div class="sys-card"><div class="sys-title">Диск</div><div class="sys-meta">Недоступно</div></div>'}
`; `;
} }
+3 -3
View File
@@ -1,5 +1,5 @@
{ {
"full": "e4704c2dc69361318d9aab490128f30c4d875bda", "full": "8bf54fb95d8d338f35bd6e628e4e21100a5bf371",
"short": "e4704c2", "short": "8bf54fb",
"date": "2026-09-24T11:21:58+03:00" "date": "2026-09-24T13:11:31+03:00"
} }
+5 -2
View File
@@ -10,10 +10,13 @@ mkdir -p backups
echo ">> Экспорт БД" echo ">> Экспорт БД"
docker compose exec -T db pg_dump -U app -d whereldo | gzip > backups/db.sql.gz docker compose exec -T db pg_dump -U app -d whereldo | gzip > backups/db.sql.gz
echo ">> Копирование фотографий" echo ">> Экспорт файлов из хранилища"
rm -rf backups/_uploads rm -rf backups/_uploads
mkdir -p backups/_uploads mkdir -p backups/_uploads
docker cp "whatido-app-1:/app/uploads/." "backups/_uploads/" docker compose exec -T app rm -rf /tmp/whatido-export
docker compose exec -T app node scripts/storage-sync.js export /tmp/whatido-export
docker cp "whatido-app-1:/tmp/whatido-export/." "backups/_uploads/"
docker compose exec -T app rm -rf /tmp/whatido-export
rm -rf backups/_uploads/.thumbs rm -rf backups/_uploads/.thumbs
echo ">> Сборка архива" echo ">> Сборка архива"
+128
View File
@@ -0,0 +1,128 @@
const path = require('path');
const { createStorage } = require('../storage');
const SKIP_PREFIXES = ['.thumbs/', '.cache/'];
function parseArgs(argv) {
const opts = { dryRun: false, deleteLocal: false, verifyOnly: false, concurrency: 3, prefix: null, help: false };
for (const arg of argv.slice(2)) {
if (arg === '--dry-run') opts.dryRun = true;
else if (arg === '--delete-local') opts.deleteLocal = true;
else if (arg === '--keep-local') opts.deleteLocal = false;
else if (arg === '--verify-only') opts.verifyOnly = true;
else if (arg.startsWith('--concurrency=')) opts.concurrency = Math.max(1, parseInt(arg.split('=')[1], 10) || 1);
else if (arg.startsWith('--prefix=')) opts.prefix = arg.split('=')[1].replace(/^\/+|\/+$/g, '');
else if (arg === '--help' || arg === '-h') opts.help = true;
}
return opts;
}
function usage() {
console.log(`Миграция uploads/ -> S3 (MinIO).
Использование:
node scripts/migrate-to-s3.js [ключи]
Ключи:
--dry-run показать, что будет загружено, ничего не менять
--verify-only только проверить наличие объектов в S3
--delete-local удалить локальные файлы после успешной проверки
--keep-local оставить локальные файлы (по умолчанию)
--concurrency=N параллельных загрузок (по умолчанию 3)
--prefix=PREFIX префикс ключей внутри бакета
Переменные окружения: STORAGE_DRIVER=s3, S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY, S3_SECRET_KEY.`);
}
async function runPool(items, limit, worker) {
const queue = items.slice();
const results = { ok: 0, skip: 0, fail: 0 };
const runners = Array.from({ length: Math.min(limit, queue.length) }, async () => {
while (queue.length) {
const item = queue.shift();
const res = await worker(item);
if (res === 'ok') results.ok++;
else if (res === 'skip') results.skip++;
else results.fail++;
}
});
await Promise.all(runners);
return results;
}
async function main() {
const opts = parseArgs(process.argv);
if (opts.help) return usage();
const dir = path.join(__dirname, '..', 'uploads');
const source = createStorage({ dir, driver: 'local' });
const target = createStorage({ driver: 's3', prefix: opts.prefix || undefined });
if (!target.isRemote()) {
console.error('STORAGE_DRIVER не s3: задайте STORAGE_DRIVER=s3 и переменные S3_*.');
process.exit(2);
}
if (!target.bucket) {
console.error('Не задан S3_BUCKET.');
process.exit(2);
}
const all = await source.listAll('');
const objects = all.filter(o => !SKIP_PREFIXES.some(p => o.key.startsWith(p)));
const totalBytes = objects.reduce((sum, o) => sum + (o.size || 0), 0);
if (!opts.dryRun) await target.ensureBucket();
console.log(`Локально: ${objects.length} файлов, ${(totalBytes / 1048576).toFixed(2)} МБ`);
console.log(`Бакет: ${target.bucket}${opts.prefix ? ' префикс ' + opts.prefix : ''} (${process.env.S3_ENDPOINT || 'endpoint по умолчанию'})`);
if (opts.dryRun) console.log('Режим --dry-run: изменения не применяются.');
const failures = [];
const result = await runPool(objects, opts.concurrency, async (obj) => {
const localPath = source.localFile(obj.key);
if (!localPath) {
failures.push(`${obj.key}: некорректный путь`);
return 'fail';
}
try {
const remote = await target.head(obj.key);
if (remote && remote.size === obj.size) {
if (opts.deleteLocal && !opts.dryRun) source.unlinkLocalOnly(obj.key);
return 'skip';
}
if (opts.verifyOnly) {
failures.push(`${obj.key}: нет объекта в S3`);
return 'fail';
}
if (opts.dryRun) {
console.log(` + ${obj.key} (${obj.size} Б)`);
return 'ok';
}
await target.putFile(obj.key, localPath);
const check = await target.head(obj.key);
if (!check || check.size !== obj.size) {
failures.push(`${obj.key}: размер не совпал после загрузки`);
return 'fail';
}
if (opts.deleteLocal) source.unlinkLocalOnly(obj.key);
return 'ok';
} catch (err) {
failures.push(`${obj.key}: ${err.message}`);
return 'fail';
}
});
const usage = await target.usage().catch(() => null);
console.log(`Загружено: ${result.ok}, уже было: ${result.skip}, ошибок: ${result.fail}`);
if (usage) console.log(`В бакете: ${usage.count} объектов, ${(usage.size_bytes / 1048576).toFixed(2)} МБ`);
if (failures.length) {
console.error('Ошибки:');
for (const f of failures.slice(0, 50)) console.error(' - ' + f);
if (failures.length > 50) console.error(` ... ещё ${failures.length - 50}`);
process.exit(1);
}
if (!opts.deleteLocal && !opts.dryRun) {
console.log('Локальные файлы оставлены. Удалить после проверки: node scripts/migrate-to-s3.js --delete-local');
}
}
main().catch(err => {
console.error('Миграция не выполнена:', err.message);
process.exit(1);
});
+11 -2
View File
@@ -28,8 +28,17 @@ echo ">> Восстановление БД (чистая схема)"
docker compose exec -T db psql -U app -d whereldo -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;' >/dev/null docker compose exec -T db psql -U app -d whereldo -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;' >/dev/null
gzip -dc "$TMP/db.sql.gz" | docker compose exec -T db psql -U app -d whereldo gzip -dc "$TMP/db.sql.gz" | docker compose exec -T db psql -U app -d whereldo
echo ">> Восстановление фотографий" echo ">> Загрузка файлов в хранилище"
docker cp "$TMP/_uploads/." "whatido-app-1:/app/uploads/" docker compose start app
for _ in $(seq 1 30); do
docker compose exec -T app true >/dev/null 2>&1 && break
sleep 1
done
docker compose exec -T app rm -rf /tmp/whatido-import
docker compose exec -T app mkdir -p /tmp/whatido-import
docker cp "$TMP/_uploads/." "whatido-app-1:/tmp/whatido-import/"
docker compose exec -T app node scripts/storage-sync.js import /tmp/whatido-import
docker compose exec -T app rm -rf /tmp/whatido-import
echo ">> Запуск приложения" echo ">> Запуск приложения"
docker compose start app docker compose start app
+39
View File
@@ -0,0 +1,39 @@
const path = require('path');
const { createStorage } = require('../storage');
function usage() {
console.log(`Экспорт/импорт файлов хранилища (для scripts/backup.sh и scripts/restore.sh).
Использование:
node scripts/storage-sync.js export <каталог> выгрузить все объекты в каталог
node scripts/storage-sync.js import <каталог> загрузить все файлы из каталога в хранилище
Работает с текущим драйвером из переменных окружения (local — ./uploads, s3 — бакет MinIO/S3).`);
}
async function main() {
const [, , mode, dirArg] = process.argv;
if (!mode || !dirArg || mode === '--help' || mode === '-h') {
usage();
return process.exit(mode ? 0 : 2);
}
const dir = path.resolve(dirArg);
const storage = createStorage({ dir: path.join(__dirname, '..', 'uploads') });
if (mode === 'export') {
const count = await storage.downloadAll(dir);
console.log(`Экспортировано объектов: ${count} -> ${dir}`);
return;
}
if (mode === 'import') {
const count = await storage.uploadTree(dir);
console.log(`Импортировано файлов: ${count} <- ${dir}`);
return;
}
usage();
process.exit(2);
}
main().catch(err => {
console.error('storage-sync failed:', err.message);
process.exit(1);
});
+196 -162
View File
@@ -8,6 +8,8 @@ const heicConvert = require('heic-convert');
const { createEntryAutoChecker, createPhotoEnhanceWorker } = require('./worker'); const { createEntryAutoChecker, createPhotoEnhanceWorker } = require('./worker');
const { createZipWriter, renderStudentReport } = require('./student-report'); const { createZipWriter, renderStudentReport } = require('./student-report');
const { createStorage, mimeFor } = require('./storage');
const https = require('https'); const https = require('https');
const path = require('path'); const path = require('path');
const fs = require('fs'); const fs = require('fs');
@@ -17,6 +19,8 @@ types.setTypeParser(1082, v => v);
const app = express(); const app = express();
const pool = new Pool({ connectionString: process.env.DATABASE_URL }); const pool = new Pool({ connectionString: process.env.DATABASE_URL });
const UPLOADS_DIR = path.join(__dirname, 'uploads');
const storage = createStorage({ dir: UPLOADS_DIR });
const pgClient = require('pg').Client; const pgClient = require('pg').Client;
const lister = new pgClient({ connectionString: process.env.DATABASE_URL }); const lister = new pgClient({ connectionString: process.env.DATABASE_URL });
@@ -257,8 +261,27 @@ app.use(helmet({
})); }));
app.use(express.json({ limit: '1mb' })); app.use(express.json({ limit: '1mb' }));
app.use(ipGuard); app.use(ipGuard);
const THUMBS_DIR = path.join(__dirname, 'uploads', '.thumbs'); app.use((req, res, next) => {
const ORIGINALS_DIR = path.join(__dirname, 'uploads', '.originals'); if (!storage.isRemote()) return next();
res.on('finish', () => {
if (res.statusCode >= 400) return;
const uploaded = [];
if (req.file) uploaded.push(req.file);
if (Array.isArray(req.files)) uploaded.push(...req.files);
else if (req.files && typeof req.files === 'object') {
for (const list of Object.values(req.files)) if (Array.isArray(list)) uploaded.push(...list);
}
for (const f of uploaded) {
if (!f || !f.filename || typeof f.path !== 'string') continue;
const abs = path.resolve(f.path);
if (!abs.startsWith(UPLOADS_DIR + path.sep)) continue;
storage.persist(f.filename, abs).catch(err => console.error('Upload persist failed:', f.filename, err.message));
}
});
next();
});
const THUMBS_DIR = path.join(UPLOADS_DIR, '.thumbs');
const ORIGINALS_DIR = path.join(UPLOADS_DIR, '.originals');
const THUMB_WIDTH = 480; const THUMB_WIDTH = 480;
let sharp = null; let sharp = null;
try { sharp = require('sharp'); } catch {} try { sharp = require('sharp'); } catch {}
@@ -266,22 +289,30 @@ if (sharp) {
try { fs.mkdirSync(THUMBS_DIR, { recursive: true }); } catch {} try { fs.mkdirSync(THUMBS_DIR, { recursive: true }); } catch {}
} }
try { fs.mkdirSync(ORIGINALS_DIR, { recursive: true }); } catch {} try { fs.mkdirSync(ORIGINALS_DIR, { recursive: true }); } catch {}
try { fs.mkdirSync(storage.cacheDir, { recursive: true }); } catch {}
function thumbFileFor(fp) { function thumbFileFor(fp) {
const base = path.basename(fp).replace(/\.[^.]+$/, '') + '.webp'; const base = path.basename(fp).replace(/\.[^.]+$/, '') + '.webp';
return path.join(THUMBS_DIR, base); return path.join(THUMBS_DIR, base);
} }
async function sendImageThumb(res, fp) { function thumbUnlinkFor(key) {
const tp = thumbFileFor(key || '');
try { if (fs.existsSync(tp)) fs.unlinkSync(tp); } catch {}
}
async function sendImageThumb(res, key) {
const local = await storage.localize(key);
if (!local) return res.status(404).end();
if (!sharp) { if (!sharp) {
res.setHeader('Cache-Control', 'public, max-age=3600'); res.setHeader('Cache-Control', 'public, max-age=3600');
return res.sendFile(fp); return res.sendFile(local);
} }
const tp = thumbFileFor(fp); const tp = thumbFileFor(key);
try { try {
if (!fs.existsSync(tp)) { if (!fs.existsSync(tp)) {
const tmp = tp + '.' + crypto.randomBytes(4).toString('hex') + '.tmp'; const tmp = tp + '.' + crypto.randomBytes(4).toString('hex') + '.tmp';
await sharp(fp).rotate().resize({ width: THUMB_WIDTH, withoutEnlargement: true }).webp({ quality: 85 }).toFile(tmp); await sharp(local).rotate().resize({ width: THUMB_WIDTH, withoutEnlargement: true }).webp({ quality: 85 }).toFile(tmp);
fs.renameSync(tmp, tp); fs.renameSync(tmp, tp);
} }
res.setHeader('Cache-Control', 'public, max-age=31536000, immutable'); res.setHeader('Cache-Control', 'public, max-age=31536000, immutable');
@@ -289,24 +320,20 @@ async function sendImageThumb(res, fp) {
} catch { } catch {
try { if (fs.existsSync(tp)) fs.unlinkSync(tp); } catch {} try { if (fs.existsSync(tp)) fs.unlinkSync(tp); } catch {}
res.setHeader('Cache-Control', 'public, max-age=3600'); res.setHeader('Cache-Control', 'public, max-age=3600');
return res.sendFile(fp); return res.sendFile(local);
} }
} }
app.get('/uploads/thumb/:name', fileLimiter, async (req, res) => { app.get('/uploads/thumb/:name', fileLimiter, async (req, res) => {
const name = req.params.name; const name = req.params.name;
if (!/^[A-Za-z0-9._-]+$/.test(name)) return res.status(400).end(); if (!/^[A-Za-z0-9._-]+$/.test(name)) return res.status(400).end();
const fp = path.join(__dirname, 'uploads', name); return sendImageThumb(res, name);
if (!fs.existsSync(fp) || !fs.statSync(fp).isFile()) return res.status(404).end();
return sendImageThumb(res, fp);
}); });
app.get('/uploads/.originals/:name', fileLimiter, async (req, res) => { app.get('/uploads/.originals/:name', fileLimiter, async (req, res) => {
const name = req.params.name; const name = req.params.name;
if (!/^[A-Za-z0-9._-]+$/.test(name)) return res.status(400).end(); if (!/^[A-Za-z0-9._-]+$/.test(name)) return res.status(400).end();
const fp = path.join(ORIGINALS_DIR, name); return sendImageThumb(res, `.originals/${name}`);
if (!fs.existsSync(fp) || !fs.statSync(fp).isFile()) return res.status(404).end();
return sendImageThumb(res, fp);
}); });
app.use((req, res, next) => { app.use((req, res, next) => {
@@ -317,7 +344,17 @@ app.use((req, res, next) => {
next(); next();
}); });
app.use('/uploads', express.static(path.join(__dirname, 'uploads'), { maxAge: '365d', immutable: true })); app.get('/uploads/*', async (req, res) => {
const key = storage.keyFromPath(req.params[0]);
if (!key) return res.status(400).end();
try {
const ok = await storage.streamTo(res, key, { cacheControl: 'public, max-age=31536000, immutable' });
if (!ok && !res.headersSent) return res.status(404).end();
} catch (e) {
console.error('Upload stream failed:', key, e.message);
if (!res.headersSent) res.status(404).end();
}
});
app.use('/vendor', express.static(path.join(__dirname, 'public', 'vendor'), { maxAge: '30d' })); app.use('/vendor', express.static(path.join(__dirname, 'public', 'vendor'), { maxAge: '30d' }));
app.use(express.static(path.join(__dirname, 'public'))); app.use(express.static(path.join(__dirname, 'public')));
@@ -580,16 +617,12 @@ async function purgeScheduledDeletions() {
return { entries: erefs.rowCount, groups: gcount }; return { entries: erefs.rowCount, groups: gcount };
} }
const UPLOADS_DIR = path.join(__dirname, 'uploads');
function safeUnlink(relPath) { function safeUnlink(relPath) {
if (!relPath || typeof relPath !== 'string') return; const key = storage.keyFromPath(relPath);
const parts = String(relPath).replace(/\\/g, '/').replace(/^\/+/, '').split('/'); if (!key) return;
if (parts[0] === 'uploads') parts.shift(); storage.unlinkLocalOnly(key);
if (!parts.length || parts.includes('..') || parts.includes('')) return; if (!storage.isRemote()) return;
const fp = path.resolve(UPLOADS_DIR, ...parts); storage.del(key).catch(err => console.error('Upload delete failed:', key, err.message));
if (fp === UPLOADS_DIR || !fp.startsWith(UPLOADS_DIR + path.sep)) return;
if (fs.existsSync(fp)) fs.unlinkSync(fp);
} }
function removeUpload(file) { function removeUpload(file) {
@@ -638,8 +671,8 @@ async function removeEntryFiles(entryId) {
} }
async function sweepOrphanedUploads() { async function sweepOrphanedUploads() {
const dir = path.join(__dirname, 'uploads'); const dir = UPLOADS_DIR;
if (!fs.existsSync(dir)) return; try { fs.mkdirSync(dir, { recursive: true }); } catch {}
const [{ rows: photos }, { rows: files }, { rows: gphotos }, { rows: ephotos }, { rows: pendingJobs }, { rows: mphotos }, { rows: sphotos }, { rows: logos }] = await Promise.all([ const [{ rows: photos }, { rows: files }, { rows: gphotos }, { rows: ephotos }, { rows: pendingJobs }, { rows: mphotos }, { rows: sphotos }, { rows: logos }] = await Promise.all([
pool.query('SELECT photo_path AS p FROM entries WHERE photo_path IS NOT NULL'), pool.query('SELECT photo_path AS p FROM entries WHERE photo_path IS NOT NULL'),
pool.query('SELECT path AS p FROM project_files'), pool.query('SELECT path AS p FROM project_files'),
@@ -651,14 +684,28 @@ async function sweepOrphanedUploads() {
pool.query(`SELECT value AS p FROM settings WHERE key = 'system_logo' AND value IS NOT NULL AND value <> ''`), pool.query(`SELECT value AS p FROM settings WHERE key = 'system_logo' AND value IS NOT NULL AND value <> ''`),
]); ]);
const refs = new Set(); const refs = new Set();
[...photos, ...files, ...gphotos, ...ephotos, ...pendingJobs, ...mphotos, ...sphotos, ...logos].forEach(r => refs.add('/' + String(r.p).replace(/^\/+/, ''))); [...photos, ...files, ...gphotos, ...ephotos, ...pendingJobs, ...mphotos, ...sphotos, ...logos].forEach(r => {
const key = storage.keyFromPath(r.p);
if (key) refs.add(key);
});
const remoteObjects = await storage.listAll('').catch(err => {
console.error('Storage list failed:', err.message);
return [];
});
for (const obj of remoteObjects) {
if (!storage.normalizeKey(obj.key) || obj.key.includes('/')) continue;
if (!refs.has(obj.key)) {
try { await storage.del(obj.key); } catch (err) { console.error('Upload delete failed:', obj.key, err.message); }
}
}
for (const f of fs.readdirSync(dir)) { for (const f of fs.readdirSync(dir)) {
const fp = path.join(dir, f); const fp = path.join(dir, f);
if (!fs.statSync(fp).isFile()) continue; if (!fs.statSync(fp).isFile()) continue;
if (!refs.has('/uploads/' + f)) { if (!refs.has(f)) {
try { fs.unlinkSync(fp); } catch {} try { fs.unlinkSync(fp); } catch {}
} }
} }
storage.pruneCache();
} }
async function ensureAuditTable() { async function ensureAuditTable() {
@@ -1704,22 +1751,7 @@ async function buildBackupArchive() {
st.rows.forEach(r => { settings[r.key] = r.value; }); st.rows.forEach(r => { settings[r.key] = r.value; });
const payload = { version: 1, created_at: new Date().toISOString(), groups: g.rows, students: s.rows, entries: e.rows, settings, project_files: pf.rows, branches: br.rows, users: us.rows, user_branches: ub.rows, group_photos: gp.rows, entry_photos: ep.rows, modules: md.rows, student_photos: sp.rows, share_links: sl.rows, photo_jobs: pj.rows }; const payload = { version: 1, created_at: new Date().toISOString(), groups: g.rows, students: s.rows, entries: e.rows, settings, project_files: pf.rows, branches: br.rows, users: us.rows, user_branches: ub.rows, group_photos: gp.rows, entry_photos: ep.rows, modules: md.rows, student_photos: sp.rows, share_links: sl.rows, photo_jobs: pj.rows };
fs.writeFileSync(path.join(staging, 'data.json'), JSON.stringify(payload)); fs.writeFileSync(path.join(staging, 'data.json'), JSON.stringify(payload));
fs.mkdirSync(path.join(staging, 'uploads'), { recursive: true }); await storage.downloadAll(path.join(staging, 'uploads'));
const dir = path.join(__dirname, 'uploads');
if (fs.existsSync(dir)) {
for (const f of fs.readdirSync(dir)) {
const fp = path.join(dir, f);
if (fs.statSync(fp).isFile() && SAFE_NAME.test(f)) fs.copyFileSync(fp, path.join(staging, 'uploads', f));
}
const orig = path.join(dir, '.originals');
if (fs.existsSync(orig)) {
fs.mkdirSync(path.join(staging, 'uploads', '.originals'), { recursive: true });
for (const f of fs.readdirSync(orig)) {
const ofp = path.join(orig, f);
if (fs.statSync(ofp).isFile() && SAFE_NAME.test(f)) fs.copyFileSync(ofp, path.join(staging, 'uploads', '.originals', f));
}
}
}
const stamp = new Date().toISOString().slice(0, 16).replace(/[:T]/g, '-'); const stamp = new Date().toISOString().slice(0, 16).replace(/[:T]/g, '-');
fs.mkdirSync(BACKUP_DIR, { recursive: true }); fs.mkdirSync(BACKUP_DIR, { recursive: true });
const outPath = path.join(BACKUP_DIR, `whatido-backup-${stamp}-${crypto.randomBytes(4).toString('hex')}.tar.gz`); const outPath = path.join(BACKUP_DIR, `whatido-backup-${stamp}-${crypto.randomBytes(4).toString('hex')}.tar.gz`);
@@ -1971,31 +2003,14 @@ app.post('/api/restore', requireAdmin, uploadBackup.single('backup'), async (req
} finally { } finally {
client.release(); client.release();
} }
const dir = path.join(__dirname, 'uploads'); fs.mkdirSync(UPLOADS_DIR, { recursive: true });
fs.mkdirSync(dir, { recursive: true });
if (legacyPhotos.length) { if (legacyPhotos.length) {
for (const p of legacyPhotos) { for (const p of legacyPhotos) {
if (!p.path || !SAFE_NAME.test(p.path)) continue; if (!p.path || !SAFE_NAME.test(p.path)) continue;
fs.writeFileSync(path.join(dir, p.path), Buffer.from(p.data, 'base64')); await storage.put(p.path, Buffer.from(p.data, 'base64'));
} }
} else { } else {
const src = path.join(staging, 'uploads'); await storage.uploadTree(path.join(staging, 'uploads'));
if (fs.existsSync(src)) {
for (const f of fs.readdirSync(src)) {
if (!SAFE_NAME.test(f)) continue;
const fp = path.join(src, f);
if (fs.statSync(fp).isFile()) fs.copyFileSync(fp, path.join(dir, f));
}
const orgSrc = path.join(src, '.originals');
if (fs.existsSync(orgSrc)) {
fs.mkdirSync(path.join(dir, '.originals'), { recursive: true });
for (const f of fs.readdirSync(orgSrc)) {
if (!SAFE_NAME.test(f)) continue;
const ofp = path.join(orgSrc, f);
if (fs.statSync(ofp).isFile()) fs.copyFileSync(ofp, path.join(dir, '.originals', f));
}
}
}
} }
fs.rmSync(staging, { recursive: true, force: true }); fs.rmSync(staging, { recursive: true, force: true });
cleanupUpload(req); cleanupUpload(req);
@@ -2315,14 +2330,17 @@ app.get('/api/share/:shareToken/files/:fileToken', fileLimiter, async (req, res)
); );
if (!rows.length) return res.status(404).json({ error: 'Not found' }); if (!rows.length) return res.status(404).json({ error: 'Not found' });
const r = rows[0]; const r = rows[0];
const fp = path.join(__dirname, r.path); const key = storage.keyFromPath(r.path);
if (!fs.existsSync(fp)) return res.status(404).json({ error: 'File missing' }); if (!key) return res.status(404).json({ error: 'File missing' });
if (isImageName(r.name)) { if (isImageName(r.name)) {
if (req.query.thumb) return sendImageThumb(res, fp); if (req.query.thumb) return sendImageThumb(res, key);
res.setHeader('Cache-Control', 'public, max-age=31536000, immutable'); const ok = await storage.streamTo(res, key, { cacheControl: 'public, max-age=31536000, immutable' });
return res.sendFile(fp); if (!ok && !res.headersSent) return res.status(404).json({ error: 'File missing' });
return;
}
if (!(await storage.streamTo(res, key, { download: true, name: r.name })) && !res.headersSent) {
return res.status(404).json({ error: 'File missing' });
} }
return res.download(fp, r.name);
}); });
app.get('/s/:token', (req, res) => { app.get('/s/:token', (req, res) => {
@@ -3332,15 +3350,14 @@ app.get('/api/export/student', requireAuth, async (req, res) => {
const seenPhotos = new Set(); const seenPhotos = new Set();
const photosBuilt = []; const photosBuilt = [];
function addPhoto(p) { async function addPhoto(p) {
if (!isSafeUploadPath(p.photo_path)) return; if (!isSafeUploadPath(p.photo_path)) return;
const stored = p.photo_path.slice('/uploads/'.length); const stored = p.photo_path.slice('/uploads/'.length);
if (seenPhotos.has(stored)) return; if (seenPhotos.has(stored)) return;
seenPhotos.add(stored); seenPhotos.add(stored);
const src = path.join(UPLOADS_DIR, stored); const data = await storage.getBuffer(stored);
if (!fs.existsSync(src)) return; if (!data) return;
const ts = p.created_at ? new Date(p.created_at) : new Date(); const ts = p.created_at ? new Date(p.created_at) : new Date();
const data = fs.readFileSync(src);
zip.addFile('photos/' + stored, data, ts); zip.addFile('photos/' + stored, data, ts);
photosBuilt.push({ photosBuilt.push({
stored, stored,
@@ -3356,13 +3373,13 @@ app.get('/api/export/student', requireAuth, async (req, res) => {
const profilePhoto = (student.profile && student.profile.photo_path) || student.photo_path; const profilePhoto = (student.profile && student.profile.photo_path) || student.photo_path;
if (profilePhoto && isSafeUploadPath(profilePhoto)) { if (profilePhoto && isSafeUploadPath(profilePhoto)) {
const stored = profilePhoto.slice('/uploads/'.length); const stored = profilePhoto.slice('/uploads/'.length);
if (fs.existsSync(path.join(UPLOADS_DIR, stored))) avatarStored = stored; if (await storage.exists(stored)) avatarStored = stored;
} }
for (const p of photosRes.rows) addPhoto(p); for (const p of photosRes.rows) await addPhoto(p);
for (const m of mainsRes.rows) addPhoto(m); for (const m of mainsRes.rows) await addPhoto(m);
photosBuilt.sort((a, b) => new Date(b.createdAt || Date.now()) - new Date(a.createdAt || Date.now())); photosBuilt.sort((a, b) => new Date(b.createdAt || Date.now()) - new Date(a.createdAt || Date.now()));
if (avatarStored) { if (avatarStored) {
if (!seenPhotos.has(avatarStored)) addPhoto({ photo_path: '/uploads/' + avatarStored, caption: 'Фото резидента', created_at: student.created_at }); if (!seenPhotos.has(avatarStored)) await addPhoto({ photo_path: '/uploads/' + avatarStored, caption: 'Фото резидента', created_at: student.created_at });
const idx = photosBuilt.findIndex(p => p.stored === avatarStored); const idx = photosBuilt.findIndex(p => p.stored === avatarStored);
if (idx > 0) photosBuilt.unshift(photosBuilt.splice(idx, 1)[0]); if (idx > 0) photosBuilt.unshift(photosBuilt.splice(idx, 1)[0]);
} }
@@ -3374,10 +3391,10 @@ app.get('/api/export/student', requireAuth, async (req, res) => {
if (!isSafeUploadPath(gp.photo_path)) continue; if (!isSafeUploadPath(gp.photo_path)) continue;
const stored = gp.photo_path.slice('/uploads/'.length); const stored = gp.photo_path.slice('/uploads/'.length);
if (seenPhotos.has(stored)) continue; if (seenPhotos.has(stored)) continue;
const src = path.join(UPLOADS_DIR, stored); const data = await storage.getBuffer(stored);
if (!fs.existsSync(src)) continue; if (!data) continue;
const ts = gp.taken_at || gp.created_at || new Date(); const ts = gp.taken_at || gp.created_at || new Date();
zip.addFile('photos/' + stored, fs.readFileSync(src), new Date(ts)); zip.addFile('photos/' + stored, data, new Date(ts));
seenPhotos.add(stored); seenPhotos.add(stored);
groupPhotosBuilt.push({ groupPhotosBuilt.push({
stored, stored,
@@ -3395,9 +3412,8 @@ app.get('/api/export/student', requireAuth, async (req, res) => {
for (const f of filesRes.rows) { for (const f of filesRes.rows) {
if (!isSafeUploadPath(f.path)) continue; if (!isSafeUploadPath(f.path)) continue;
const stored = f.path.slice('/uploads/'.length); const stored = f.path.slice('/uploads/'.length);
const src = path.join(UPLOADS_DIR, stored); const data = await storage.getBuffer(stored);
if (!fs.existsSync(src)) continue; if (!data) continue;
const data = fs.readFileSync(src);
let base = String(f.name || 'file').replace(/[\\/:*?"<>|]/g, '_').replace(/^[.\s]+/, '').slice(0, 120) || 'file'; let base = String(f.name || 'file').replace(/[\\/:*?"<>|]/g, '_').replace(/^[.\s]+/, '').slice(0, 120) || 'file';
const ext = path.extname(base); const ext = path.extname(base);
const stem = ext ? base.slice(0, -ext.length) : base; const stem = ext ? base.slice(0, -ext.length) : base;
@@ -3700,11 +3716,10 @@ app.get('/api/files', requireAuth, async (req, res) => {
const off = parseInt(offset, 10); const off = parseInt(offset, 10);
if (off > 0) { qparams.push(off); q += ` OFFSET $${qparams.length}`; } if (off > 0) { qparams.push(off); q += ` OFFSET $${qparams.length}`; }
const { rows } = await pool.query(q, qparams); const { rows } = await pool.query(q, qparams);
const files = rows.map(r => { const files = await Promise.all(rows.map(async r => {
let size = 0; const size = await storage.sizeOf(r.path);
try { size = fs.statSync(path.join(__dirname, r.path)).size; } catch {}
return { id: r.id, token: r.token, name: r.name, student_name: r.student_name, group_name: r.group_name, created_at: r.created_at, size }; return { id: r.id, token: r.token, name: r.name, student_name: r.student_name, group_name: r.group_name, created_at: r.created_at, size };
}); }));
res.json({ files, total }); res.json({ files, total });
}); });
@@ -3728,11 +3743,10 @@ app.get('/api/files/detached', requireAdmin, async (req, res) => {
const off = parseInt(offset, 10); const off = parseInt(offset, 10);
if (off > 0) { qparams.push(off); q += ` OFFSET $${qparams.length}`; } if (off > 0) { qparams.push(off); q += ` OFFSET $${qparams.length}`; }
const { rows } = await pool.query(q, qparams); const { rows } = await pool.query(q, qparams);
const files = rows.map(r => { const files = await Promise.all(rows.map(async r => {
let size = 0; const size = await storage.sizeOf(r.path);
try { size = fs.statSync(path.join(__dirname, r.path)).size; } catch {}
return { id: r.id, token: r.token, name: r.name, created_at: r.created_at, size }; return { id: r.id, token: r.token, name: r.name, created_at: r.created_at, size };
}); }));
res.json({ files, total }); res.json({ files, total });
}); });
@@ -3783,14 +3797,17 @@ app.get('/api/files/:token', fileLimiter, async (req, res) => {
const { rows } = await pool.query('SELECT path, name FROM project_files WHERE token = $1', [req.params.token]); const { rows } = await pool.query('SELECT path, name FROM project_files WHERE token = $1', [req.params.token]);
if (!rows.length) return res.status(404).json({ error: 'Not found' }); if (!rows.length) return res.status(404).json({ error: 'Not found' });
const r = rows[0]; const r = rows[0];
const fp = path.join(__dirname, r.path); const key = storage.keyFromPath(r.path);
if (!fs.existsSync(fp)) return res.status(404).json({ error: 'File missing' }); if (!key) return res.status(404).json({ error: 'File missing' });
if (isImageName(r.name)) { if (isImageName(r.name)) {
if (req.query.thumb) return sendImageThumb(res, fp); if (req.query.thumb) return sendImageThumb(res, key);
res.setHeader('Cache-Control', 'public, max-age=31536000, immutable'); const ok = await storage.streamTo(res, key, { cacheControl: 'public, max-age=31536000, immutable' });
return res.sendFile(fp); if (!ok && !res.headersSent) return res.status(404).json({ error: 'File missing' });
return;
}
if (!(await storage.streamTo(res, key, { download: true, name: r.name })) && !res.headersSent) {
return res.status(404).json({ error: 'File missing' });
} }
return res.download(fp, r.name);
}); });
app.get('/api/stats', requireAuth, async (req, res) => { app.get('/api/stats', requireAuth, async (req, res) => {
@@ -3871,36 +3888,23 @@ app.get('/api/system-info', requireAdmin, async (_, res) => {
WHERE photo_path IS NOT NULL AND deleted_at IS NULL WHERE photo_path IS NOT NULL AND deleted_at IS NULL
`); `);
function sumPhotoSizes(paths) { async function sumPhotoSizes(paths) {
let bytes = 0; let bytes = 0;
for (const p of paths) { for (const p of paths) {
if (!p) continue; if (!p) continue;
const fp = path.join(__dirname, p); bytes += await storage.sizeOf(p);
try {
const st = fs.statSync(fp);
if (st.isFile()) bytes += st.size;
} catch {}
} }
return bytes; return bytes;
} }
const groupPhotoSizes = await pool.query('SELECT photo_path FROM group_photos'); const groupPhotoSizes = await pool.query('SELECT photo_path FROM group_photos');
const entryPhotoSizes = await pool.query('SELECT photo_path FROM entries WHERE photo_path IS NOT NULL AND deleted_at IS NULL'); const entryPhotoSizes = await pool.query('SELECT photo_path FROM entries WHERE photo_path IS NOT NULL AND deleted_at IS NULL');
const groupPhotoBytes = sumPhotoSizes(groupPhotoSizes.rows.map(r => r.photo_path)); const groupPhotoBytes = await sumPhotoSizes(groupPhotoSizes.rows.map(r => r.photo_path));
const entryPhotoBytes = sumPhotoSizes(entryPhotoSizes.rows.map(r => r.photo_path)); const entryPhotoBytes = await sumPhotoSizes(entryPhotoSizes.rows.map(r => r.photo_path));
const uploadsDir = path.join(__dirname, 'uploads'); const usage = await storage.usage();
let uploadsSize = 0; const uploadsSize = usage.size_bytes;
let uploadsCount = 0; const uploadsCount = usage.count;
if (fs.existsSync(uploadsDir)) {
for (const f of fs.readdirSync(uploadsDir)) {
const fp = path.join(uploadsDir, f);
if (fs.statSync(fp).isFile()) {
uploadsCount++;
uploadsSize += fs.statSync(fp).size;
}
}
}
const diskInfo = getDiskInfo(); const diskInfo = getDiskInfo();
@@ -3929,6 +3933,14 @@ app.get('/api/system-info', requireAdmin, async (_, res) => {
size: formatBytes(uploadsSize), size: formatBytes(uploadsSize),
size_bytes: uploadsSize, size_bytes: uploadsSize,
}, },
storage: {
driver: storage.isRemote() ? 's3' : 'local',
bucket: usage.bucket,
prefix: usage.prefix,
count: usage.count,
size: formatBytes(usage.size_bytes),
size_bytes: usage.size_bytes,
},
disk: diskInfo, disk: diskInfo,
}; };
}); });
@@ -4285,26 +4297,26 @@ async function swapEntryPhotoFiles(req, entryId, oldPath, newPath, { keepOrigina
let originalPath = null; let originalPath = null;
const { rows: prevRows } = await pool.query('SELECT photo_original_path FROM entries WHERE id = $1', [entryId]); const { rows: prevRows } = await pool.query('SELECT photo_original_path FROM entries WHERE id = $1', [entryId]);
const prevOriginal = prevRows.length ? prevRows[0].photo_original_path : null; const prevOriginal = prevRows.length ? prevRows[0].photo_original_path : null;
const oldAbs = path.join(UPLOADS_DIR, String(oldPath).replace(/^\/+/, '').replace(/^uploads\//, '')); const oldKey = storage.keyFromPath(oldPath);
if (prevOriginal) { if (prevOriginal) {
originalPath = prevOriginal; originalPath = prevOriginal;
try { if (fs.existsSync(oldAbs)) fs.unlinkSync(oldAbs); } catch {} if (oldKey) safeUnlink(oldKey);
} else if (keepOriginal && fs.existsSync(oldAbs)) { } else if (keepOriginal && oldKey && (await storage.exists(oldKey))) {
try { try {
const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg'); const backupKey = `.originals/${crypto.randomBytes(12).toString('hex')}${path.extname(oldKey) || '.jpg'}`;
const backupPath = path.join(ORIGINALS_DIR, backupName); if (await storage.copyObject(oldKey, backupKey)) {
fs.renameSync(oldAbs, backupPath); originalPath = `/uploads/${backupKey}`;
originalPath = `/uploads/.originals/${backupName}`; safeUnlink(oldKey);
}
} catch (e) { } catch (e) {
console.error('photo original backup failed:', e); console.error('photo original backup failed:', e);
} }
} else { } else if (oldKey) {
try { if (fs.existsSync(oldAbs)) fs.unlinkSync(oldAbs); } catch {} safeUnlink(oldKey);
} }
await pool.query('UPDATE entries SET photo_path = $1, photo_original_path = $2 WHERE id = $3', [newPath, originalPath, entryId]); await pool.query('UPDATE entries SET photo_path = $1, photo_original_path = $2 WHERE id = $3', [newPath, originalPath, entryId]);
await pool.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [newPath, entryId, oldPath]); await pool.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [newPath, entryId, oldPath]);
const oldThumb = path.join('uploads', '.thumbs', path.basename(oldPath).replace(/\.[^.]+$/, '') + '.webp'); thumbUnlinkFor(oldPath);
safeUnlink(oldThumb);
await pool.query( await pool.query(
'INSERT INTO photo_jobs (entry_id, action, status, before_path, after_path, finished_at) VALUES ($1, $2, $3, $4, $5, now())', 'INSERT INTO photo_jobs (entry_id, action, status, before_path, after_path, finished_at) VALUES ($1, $2, $3, $4, $5, now())',
[entryId, action, 'done', originalPath, newPath] [entryId, action, 'done', originalPath, newPath]
@@ -4343,7 +4355,9 @@ app.put('/api/entries/:id/photo/enhance', requireAuth, (req, res, next) => {
const saturate = clampEnhanceParam(p.saturate, 0, 300, 100); const saturate = clampEnhanceParam(p.saturate, 0, 300, 100);
const sharpAmt = clampEnhanceParam(p.sharp, 0, 100, 0); const sharpAmt = clampEnhanceParam(p.sharp, 0, 100, 0);
const denoise = clampEnhanceParam(p.denoise, 0, 100, 0); const denoise = clampEnhanceParam(p.denoise, 0, 100, 0);
const srcPath = path.join(__dirname, oldPath.replace(/^\/+/, '')); const srcKey = storage.keyFromPath(oldPath);
const srcPath = srcKey ? await storage.localize(srcKey) : null;
if (!srcPath) return res.status(400).json({ error: 'Файл фото не найден' });
let pipeline = sharp(srcPath).rotate(); let pipeline = sharp(srcPath).rotate();
if (denoise > 0) pipeline = pipeline.median(denoise > 70 ? 5 : 3); if (denoise > 0) pipeline = pipeline.median(denoise > 70 ? 5 : 3);
pipeline = pipeline.modulate({ brightness: brightness / 100, saturation: saturate / 100 }); pipeline = pipeline.modulate({ brightness: brightness / 100, saturation: saturate / 100 });
@@ -4355,8 +4369,9 @@ app.put('/api/entries/:id/photo/enhance', requireAuth, (req, res, next) => {
pipeline = pipeline.sharpen({ sigma: 0.5 + (sharpAmt / 100) * 1.5, m1: 0, m2: 1 + sharpAmt / 50 }); pipeline = pipeline.sharpen({ sigma: 0.5 + (sharpAmt / 100) * 1.5, m1: 0, m2: 1 + sharpAmt / 50 });
} }
const newName = crypto.randomBytes(12).toString('hex') + '.jpg'; const newName = crypto.randomBytes(12).toString('hex') + '.jpg';
const outPath = path.join(__dirname, 'uploads', newName); const outPath = path.join(UPLOADS_DIR, newName);
await pipeline.jpeg({ quality: 92, mozjpeg: true }).toFile(outPath); await pipeline.jpeg({ quality: 92, mozjpeg: true }).toFile(outPath);
await storage.persist(newName, outPath);
newPath = `/uploads/${newName}`; newPath = `/uploads/${newName}`;
engine = 'sharp'; engine = 'sharp';
} else { } else {
@@ -4436,8 +4451,7 @@ app.post('/api/entries/:id/photo/jobs/:jobId/apply', requireAuth, async (req, re
await client.query('ROLLBACK'); await client.query('ROLLBACK');
return res.status(400).json({ error: 'Некорректный путь результата' }); return res.status(400).json({ error: 'Некорректный путь результата' });
} }
const afterAbs = path.join(UPLOADS_DIR, String(job.after_path).replace(/^\/+/, '').replace(/^uploads\//, '')); if (!(await storage.exists(job.after_path))) {
if (!fs.existsSync(afterAbs)) {
await client.query('ROLLBACK'); await client.query('ROLLBACK');
return res.status(400).json({ error: 'Файл результата не найден' }); return res.status(400).json({ error: 'Файл результата не найден' });
} }
@@ -4451,21 +4465,22 @@ app.post('/api/entries/:id/photo/jobs/:jobId/apply', requireAuth, async (req, re
} }
let originalPath = entryRows[0].photo_original_path; let originalPath = entryRows[0].photo_original_path;
let beforePath = originalPath; let beforePath = originalPath;
const oldAbs = path.join(UPLOADS_DIR, String(oldPath || '').replace(/^\/+/, '').replace(/^uploads\//, '')); const oldKey = oldPath ? storage.keyFromPath(oldPath) : null;
if (oldPath && fs.existsSync(oldAbs)) { if (oldKey && (await storage.exists(oldKey))) {
try { try {
const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg'); const backupKey = `.originals/${crypto.randomBytes(12).toString('hex')}${path.extname(oldKey) || '.jpg'}`;
fs.renameSync(oldAbs, path.join(ORIGINALS_DIR, backupName)); if (await storage.copyObject(oldKey, backupKey)) {
beforePath = `/uploads/.originals/${backupName}`; beforePath = `/uploads/${backupKey}`;
if (!originalPath) originalPath = beforePath; if (!originalPath) originalPath = beforePath;
safeUnlink(oldKey);
}
} catch (e) { } catch (e) {
console.error('photo apply backup failed:', e); console.error('photo apply backup failed:', e);
} }
} }
await client.query('UPDATE entries SET photo_path = $1, photo_original_path = $2 WHERE id = $3', [job.after_path, originalPath, req.params.id]); await client.query('UPDATE entries SET photo_path = $1, photo_original_path = $2 WHERE id = $3', [job.after_path, originalPath, req.params.id]);
await client.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [job.after_path, req.params.id, oldPath]); await client.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [job.after_path, req.params.id, oldPath]);
const oldThumb = path.join('uploads', '.thumbs', path.basename(oldPath || '').replace(/\.[^.]+$/, '') + '.webp'); thumbUnlinkFor(oldPath);
safeUnlink(oldThumb);
await client.query(`UPDATE photo_jobs SET applied = true, before_path = COALESCE(before_path, $1) WHERE id = $2`, [beforePath, jobId]); await client.query(`UPDATE photo_jobs SET applied = true, before_path = COALESCE(before_path, $1) WHERE id = $2`, [beforePath, jobId]);
await client.query('COMMIT'); await client.query('COMMIT');
await logAudit(req, 'entry.photo.apply', { entry_id: req.params.id, job_id: jobId, before_path: beforePath, after_path: job.after_path }); await logAudit(req, 'entry.photo.apply', { entry_id: req.params.id, job_id: jobId, before_path: beforePath, after_path: job.after_path });
@@ -4544,28 +4559,32 @@ app.post('/api/entries/:id/photo/jobs/:jobId/rollback', requireAuth, async (req,
} }
const beforeName = path.basename(job.before_path); const beforeName = path.basename(job.before_path);
if (!/^[A-Za-z0-9._-]+$/.test(beforeName)) return res.status(400).json({ error: 'Некорректный путь' }); if (!/^[A-Za-z0-9._-]+$/.test(beforeName)) return res.status(400).json({ error: 'Некорректный путь' });
const beforeAbs = path.join(ORIGINALS_DIR, beforeName); const beforeKey = `.originals/${beforeName}`;
if (!fs.existsSync(beforeAbs)) return res.status(400).json({ error: 'Файл версии не найден' }); if (!(await storage.exists(beforeKey))) return res.status(400).json({ error: 'Файл версии не найден' });
const { rows: cur } = await pool.query('SELECT photo_path FROM entries WHERE id = $1', [req.params.id]); const { rows: cur } = await pool.query('SELECT photo_path FROM entries WHERE id = $1', [req.params.id]);
if (!cur.length) return res.status(404).json({ error: 'Запись не найдена' }); if (!cur.length) return res.status(404).json({ error: 'Запись не найдена' });
const oldPath = cur[0].photo_path; const oldPath = cur[0].photo_path;
if (!oldPath) return res.status(400).json({ error: 'У записи нет фото' }); if (!oldPath) return res.status(400).json({ error: 'У записи нет фото' });
const ext = path.extname(beforeName) || '.jpg'; const ext = path.extname(beforeName) || '.jpg';
const newPath = `/uploads/${crypto.randomBytes(12).toString('hex')}${ext}`; const newKey = `${crypto.randomBytes(12).toString('hex')}${ext}`;
fs.copyFileSync(beforeAbs, path.join(UPLOADS_DIR, path.basename(newPath))); const newPath = `/uploads/${newKey}`;
const oldAbs = path.join(UPLOADS_DIR, String(oldPath).replace(/^\/+/, '').replace(/^uploads\//, '')); if (!(await storage.copyObject(beforeKey, newKey))) {
return res.status(400).json({ error: 'Файл версии не найден' });
}
const oldKey = storage.keyFromPath(oldPath);
let rollbackBefore = null; let rollbackBefore = null;
if (fs.existsSync(oldAbs)) { if (oldKey && (await storage.exists(oldKey))) {
try { try {
const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg'); const backupKey = `.originals/${crypto.randomBytes(12).toString('hex')}${path.extname(oldKey) || '.jpg'}`;
fs.renameSync(oldAbs, path.join(ORIGINALS_DIR, backupName)); if (await storage.copyObject(oldKey, backupKey)) {
rollbackBefore = `/uploads/.originals/${backupName}`; rollbackBefore = `/uploads/${backupKey}`;
safeUnlink(oldKey);
}
} catch (e) { } catch (e) {
console.error('photo rollback backup failed:', e); console.error('photo rollback backup failed:', e);
} }
} }
const oldThumb = path.join('uploads', '.thumbs', path.basename(oldPath).replace(/\.[^.]+$/, '') + '.webp'); thumbUnlinkFor(oldPath);
safeUnlink(oldThumb);
await pool.query('UPDATE entries SET photo_path = $1 WHERE id = $2', [newPath, req.params.id]); await pool.query('UPDATE entries SET photo_path = $1 WHERE id = $2', [newPath, req.params.id]);
await pool.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [newPath, req.params.id, oldPath]); await pool.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [newPath, req.params.id, oldPath]);
await pool.query( await pool.query(
@@ -4606,26 +4625,29 @@ app.post('/api/entries/:id/photo/restore-original', requireAuth, async (req, res
if (!origPath) return res.status(400).json({ error: 'Оригинал не сохранён' }); if (!origPath) return res.status(400).json({ error: 'Оригинал не сохранён' });
const origName = path.basename(origPath); const origName = path.basename(origPath);
if (!/^[A-Za-z0-9._-]+$/.test(origName)) return res.status(400).json({ error: 'Некорректный путь оригинала' }); if (!/^[A-Za-z0-9._-]+$/.test(origName)) return res.status(400).json({ error: 'Некорректный путь оригинала' });
const origAbs = path.join(ORIGINALS_DIR, origName); const origKey = `.originals/${origName}`;
if (!fs.existsSync(origAbs)) return res.status(400).json({ error: 'Файл оригинала не найден' }); if (!(await storage.exists(origKey))) return res.status(400).json({ error: 'Файл оригинала не найден' });
const newPath = `/uploads/${crypto.randomBytes(12).toString('hex')}${path.extname(origName) || '.jpg'}`; const newKey = `${crypto.randomBytes(12).toString('hex')}${path.extname(origName) || '.jpg'}`;
fs.renameSync(origAbs, path.join(UPLOADS_DIR, path.basename(newPath))); const newPath = `/uploads/${newKey}`;
if (!(await storage.copyObject(origKey, newKey))) return res.status(400).json({ error: 'Файл оригинала не найден' });
await storage.del(origKey);
const oldPath = rows[0].photo_path; const oldPath = rows[0].photo_path;
await pool.query('UPDATE entries SET photo_path = $1, photo_original_path = NULL WHERE id = $2', [newPath, req.params.id]); await pool.query('UPDATE entries SET photo_path = $1, photo_original_path = NULL WHERE id = $2', [newPath, req.params.id]);
await pool.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [newPath, req.params.id, oldPath]); await pool.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [newPath, req.params.id, oldPath]);
const oldAbs = path.join(UPLOADS_DIR, String(oldPath).replace(/^\/+/, '').replace(/^uploads\//, '')); const oldKey = storage.keyFromPath(oldPath);
let beforePath = null; let beforePath = null;
if (fs.existsSync(oldAbs)) { if (oldKey && (await storage.exists(oldKey))) {
try { try {
const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg'); const backupKey = `.originals/${crypto.randomBytes(12).toString('hex')}${path.extname(oldKey) || '.jpg'}`;
fs.renameSync(oldAbs, path.join(ORIGINALS_DIR, backupName)); if (await storage.copyObject(oldKey, backupKey)) {
beforePath = `/uploads/.originals/${backupName}`; beforePath = `/uploads/${backupKey}`;
safeUnlink(oldKey);
}
} catch (e) { } catch (e) {
console.error('photo original backup failed:', e); console.error('photo original backup failed:', e);
} }
} }
const oldThumb = path.join('uploads', '.thumbs', path.basename(oldPath).replace(/\.[^.]+$/, '') + '.webp'); thumbUnlinkFor(oldPath);
safeUnlink(oldThumb);
await pool.query( await pool.query(
'INSERT INTO photo_jobs (entry_id, action, status, before_path, after_path, error, finished_at) VALUES ($1, $2, $3, $4, $5, NULL, now())', 'INSERT INTO photo_jobs (entry_id, action, status, before_path, after_path, error, finished_at) VALUES ($1, $2, $3, $4, $5, NULL, now())',
[req.params.id, 'restore', 'done', beforePath, newPath] [req.params.id, 'restore', 'done', beforePath, newPath]
@@ -5250,8 +5272,19 @@ if (fs.existsSync(certPath) && fs.existsSync(keyPath)) {
try { await pool.query(`INSERT INTO settings (key, value) VALUES ('camera_enabled', 'true') ON CONFLICT (key) DO NOTHING`); } catch (err) { console.error('Camera setting:', err); } try { await pool.query(`INSERT INTO settings (key, value) VALUES ('camera_enabled', 'true') ON CONFLICT (key) DO NOTHING`); } catch (err) { console.error('Camera setting:', err); }
try { await pool.query(`INSERT INTO settings (key, value) VALUES ('trash_purge_days', '30') ON CONFLICT (key) DO NOTHING`); } catch (err) { console.error('Trash purge days setting:', err); } try { await pool.query(`INSERT INTO settings (key, value) VALUES ('trash_purge_days', '30') ON CONFLICT (key) DO NOTHING`); } catch (err) { console.error('Trash purge days setting:', err); }
try { await sweepOrphanedUploads(); } catch (err) { console.error('Upload sweep:', err); } try { await sweepOrphanedUploads(); } catch (err) { console.error('Upload sweep:', err); }
if (storage.isRemote()) {
try {
await storage.ensureBucket();
console.log('S3 bucket ready:', storage.bucket);
} catch (err) {
console.error('S3 bucket init:', err.message);
}
}
try { await purgeScheduledDeletions(); } catch (err) { console.error('Trash purge:', err); } try { await purgeScheduledDeletions(); } catch (err) { console.error('Trash purge:', err); }
setInterval(() => { purgeScheduledDeletions().catch(err => console.error('Trash purge:', err)); }, 60 * 60 * 1000).unref(); setInterval(() => { purgeScheduledDeletions().catch(err => console.error('Trash purge:', err)); }, 60 * 60 * 1000).unref();
setInterval(() => {
try { storage.pruneCache(); } catch (err) { console.error('Cache prune:', err); }
}, 60 * 60 * 1000).unref();
entryAutoChecker = createEntryAutoChecker({ pool, getSetting, logAudit, aiUrl: AI_URL, defaultPrompt: AI_DEFAULT_PROMPT }); entryAutoChecker = createEntryAutoChecker({ pool, getSetting, logAudit, aiUrl: AI_URL, defaultPrompt: AI_DEFAULT_PROMPT });
entryAutoChecker.start(); entryAutoChecker.start();
console.log('AI auto-check worker started'); console.log('AI auto-check worker started');
@@ -5264,6 +5297,7 @@ if (fs.existsSync(certPath) && fs.existsSync(keyPath)) {
photoAiUrl: PHOTO_AI_URL, photoAiUrl: PHOTO_AI_URL,
uploadsDir: UPLOADS_DIR, uploadsDir: UPLOADS_DIR,
originalsDir: ORIGINALS_DIR, originalsDir: ORIGINALS_DIR,
storage,
}); });
photoWorker.start(); photoWorker.start();
console.log('Photo enhance worker started'); console.log('Photo enhance worker started');
+551
View File
@@ -0,0 +1,551 @@
const fs = require('fs');
const path = require('path');
const { pipeline } = require('stream/promises');
const {
S3Client,
PutObjectCommand,
GetObjectCommand,
HeadObjectCommand,
DeleteObjectCommand,
CopyObjectCommand,
ListObjectsV2Command,
HeadBucketCommand,
CreateBucketCommand,
} = require('@aws-sdk/client-s3');
const MIME_TYPES = {
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.jfif': 'image/jpeg',
'.png': 'image/png',
'.gif': 'image/gif',
'.webp': 'image/webp',
'.bmp': 'image/bmp',
'.avif': 'image/avif',
'.ico': 'image/x-icon',
'.heic': 'image/heic',
'.heif': 'image/heif',
'.pdf': 'application/pdf',
'.txt': 'text/plain; charset=utf-8',
'.md': 'text/markdown; charset=utf-8',
'.html': 'text/html; charset=utf-8',
'.htm': 'text/html; charset=utf-8',
'.zip': 'application/zip',
'.rar': 'application/vnd.rar',
'.7z': 'application/x-7z-compressed',
'.doc': 'application/msword',
'.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
};
const SAFE_SEGMENT = /^[\w,.()-]+$/;
const NON_OBJECT_PREFIXES = ['.thumbs/', '.cache/'];
function mimeFor(name) {
const m = String(name || '').toLowerCase().match(/\.[a-z0-9]+$/);
return (m && MIME_TYPES[m[0]]) || 'application/octet-stream';
}
function normalizeKey(input) {
if (typeof input !== 'string' || !input) return null;
let key = input.replace(/\\/g, '/').replace(/^\/+/, '');
if (key.startsWith('uploads/')) key = key.slice('uploads/'.length);
if (!key || key.length > 255 || key.includes('..')) return null;
const segments = key.split('/');
if (segments.some(seg => !SAFE_SEGMENT.test(seg))) return null;
return key;
}
function isObjectKey(key) {
return !NON_OBJECT_PREFIXES.some(p => key.startsWith(p));
}
function createStorage(options = {}) {
const driver = options.driver || process.env.STORAGE_DRIVER || 'local';
const dir = path.resolve(options.dir || path.join(__dirname, 'uploads'));
const cacheDir = path.resolve(options.cacheDir || path.join(dir, '.cache'));
const localFallback = options.localFallback !== undefined
? !!options.localFallback
: process.env.STORAGE_LOCAL_FALLBACK !== '0';
const keepLocal = options.keepLocal !== undefined
? !!options.keepLocal
: process.env.STORAGE_KEEP_LOCAL === '1';
const bucket = options.bucket || process.env.S3_BUCKET || '';
const prefix = String(options.prefix !== undefined ? options.prefix : process.env.S3_PREFIX || '').replace(/^\/+|\/+$/g, '');
const cacheMaxAgeMs = Math.max(0, parseInt(options.cacheMaxAgeHours || process.env.STORAGE_CACHE_MAX_AGE_HOURS || '168', 10) || 0) * 3600 * 1000;
const remote = driver === 's3';
const client = remote
? new S3Client({
region: options.region || process.env.S3_REGION || 'us-east-1',
endpoint: options.endpoint || process.env.S3_ENDPOINT || undefined,
forcePathStyle: options.forcePathStyle !== undefined
? !!options.forcePathStyle
: process.env.S3_FORCE_PATH_STYLE !== '0',
credentials: {
accessKeyId: options.accessKey || process.env.S3_ACCESS_KEY || '',
secretAccessKey: options.secretKey || process.env.S3_SECRET_KEY || '',
},
})
: null;
function isRemote() {
return remote;
}
function objectKey(key) {
const k = normalizeKey(key);
if (!k) return null;
return prefix ? `${prefix}/${k}` : k;
}
function localFile(key) {
const k = normalizeKey(key);
if (!k) return null;
const abs = path.resolve(dir, ...k.split('/'));
if (abs !== dir && !abs.startsWith(dir + path.sep)) return null;
return abs;
}
function cacheFile(key) {
const k = normalizeKey(key);
if (!k) return null;
return path.join(cacheDir, ...k.split('/'));
}
function publicPath(key) {
const k = normalizeKey(key);
return k ? `/uploads/${k}` : null;
}
function keyFromPath(p) {
return normalizeKey(p);
}
function localExists(key) {
const fp = localFile(key);
if (!fp) return false;
try {
return fs.statSync(fp).isFile();
} catch {
return false;
}
}
function unlinkLocalOnly(key) {
for (const fp of [localFile(key), cacheFile(key)]) {
if (!fp) continue;
try {
if (fs.existsSync(fp)) fs.unlinkSync(fp);
} catch {}
}
}
function isMissingError(err) {
if (!err) return false;
const code = err.name || err.Code || err.code;
const status = err.$metadata && err.$metadata.httpStatusCode;
return code === 'NoSuchKey' || code === 'NotFound' || code === 'ENOENT' || status === 404;
}
async function put(key, body, opts = {}) {
const objKey = objectKey(key);
if (!objKey) return false;
if (!remote) {
const fp = localFile(key);
if (!fp) return false;
fs.mkdirSync(path.dirname(fp), { recursive: true });
if (Buffer.isBuffer(body) || typeof body === 'string') fs.writeFileSync(fp, body);
else await pipeline(body, fs.createWriteStream(fp));
return true;
}
await client.send(new PutObjectCommand({
Bucket: bucket,
Key: objKey,
Body: body,
ContentType: opts.contentType || mimeFor(key),
ContentLength: opts.contentLength,
}));
return true;
}
async function putFile(key, localPath, opts = {}) {
const objKey = objectKey(key);
if (!objKey) return false;
let size = null;
try {
size = fs.statSync(localPath).size;
} catch {
return false;
}
if (!remote) {
const fp = localFile(key);
if (!fp) return false;
fs.mkdirSync(path.dirname(fp), { recursive: true });
if (path.resolve(localPath) !== fp) fs.copyFileSync(localPath, fp);
return true;
}
await client.send(new PutObjectCommand({
Bucket: bucket,
Key: objKey,
Body: fs.createReadStream(localPath),
ContentLength: size,
ContentType: opts.contentType || mimeFor(key),
}));
return true;
}
async function head(key) {
const objKey = objectKey(key);
if (!objKey) return null;
if (!remote) {
const fp = localFile(key);
if (!fp) return null;
try {
const st = fs.statSync(fp);
if (!st.isFile()) return null;
return { key: normalizeKey(key), size: st.size, contentType: mimeFor(key), etag: null };
} catch {
return null;
}
}
try {
const out = await client.send(new HeadObjectCommand({ Bucket: bucket, Key: objKey }));
return {
key: normalizeKey(key),
size: out.ContentLength || 0,
contentType: out.ContentType || mimeFor(key),
etag: out.ETag ? out.ETag.replace(/"/g, '') : null,
};
} catch (err) {
if (isMissingError(err)) return null;
throw err;
}
}
async function exists(key) {
if (localFallback && localExists(key)) return true;
return !!(await head(key));
}
async function sizeOf(key) {
const info = await head(key);
return info ? info.size : 0;
}
async function getStream(key) {
const objKey = objectKey(key);
if (!objKey) return null;
if (!remote) {
const fp = localFile(key);
if (!fp || !fs.existsSync(fp)) return null;
return { stream: fs.createReadStream(fp), contentLength: fs.statSync(fp).size, contentType: mimeFor(key) };
}
try {
const out = await client.send(new GetObjectCommand({ Bucket: bucket, Key: objKey }));
return { stream: out.Body, contentLength: out.ContentLength || 0, contentType: out.ContentType || mimeFor(key) };
} catch (err) {
if (isMissingError(err)) return null;
throw err;
}
}
async function getBuffer(key) {
const fp = await localize(key);
if (fp) return fs.readFileSync(fp);
return null;
}
async function del(key) {
const k = normalizeKey(key);
if (!k) return false;
unlinkLocalOnly(k);
if (!remote) return true;
const objKey = objectKey(k);
try {
await client.send(new DeleteObjectCommand({ Bucket: bucket, Key: objKey }));
} catch (err) {
if (!isMissingError(err)) throw err;
}
return true;
}
async function copyObject(srcKey, dstKey) {
const src = normalizeKey(srcKey);
const dst = normalizeKey(dstKey);
if (!src || !dst) return false;
if (!remote) {
const from = localFile(src);
const to = localFile(dst);
if (!from || !to || !fs.existsSync(from)) return false;
fs.mkdirSync(path.dirname(to), { recursive: true });
fs.copyFileSync(from, to);
return true;
}
try {
await client.send(new CopyObjectCommand({
Bucket: bucket,
Key: objectKey(dst),
CopySource: `${bucket}/${objectKey(src)}`,
ContentType: mimeFor(dst),
MetadataDirective: 'REPLACE',
}));
return true;
} catch (err) {
if (isMissingError(err)) return false;
throw err;
}
}
async function listAll(listPrefix = '') {
const out = [];
if (!remote) {
const base = listPrefix ? localFile(listPrefix) : dir;
if (!base || !fs.existsSync(base)) return out;
const walk = (abs, rel) => {
for (const name of fs.readdirSync(abs)) {
const childAbs = path.join(abs, name);
const childRel = rel ? `${rel}/${name}` : name;
let st;
try {
st = fs.statSync(childAbs);
} catch {
continue;
}
if (st.isDirectory()) walk(childAbs, childRel);
else out.push({ key: childRel, size: st.size });
}
};
walk(base, listPrefix ? normalizeKey(listPrefix) || '' : '');
return out;
}
let token = null;
const fullPrefix = prefix ? `${prefix}/${listPrefix}` : listPrefix;
do {
const res = await client.send(new ListObjectsV2Command({
Bucket: bucket,
Prefix: fullPrefix || undefined,
ContinuationToken: token || undefined,
}));
for (const item of res.Contents || []) {
const key = prefix ? String(item.Key).slice(prefix.length + 1) : String(item.Key);
out.push({ key, size: item.Size || 0 });
}
token = res.IsTruncated ? res.NextContinuationToken : null;
} while (token);
return out;
}
async function localize(key, opts = {}) {
const k = normalizeKey(key);
if (!k) return null;
if (localExists(k)) return localFile(k);
if (!remote) return null;
const cf = cacheFile(k);
if (cf && fs.existsSync(cf)) {
const info = await head(k);
if (info && info.size === fs.statSync(cf).size) return cf;
try { fs.unlinkSync(cf); } catch {}
}
const source = await getStream(k);
if (!source) return null;
fs.mkdirSync(path.dirname(cf), { recursive: true });
const tmp = `${cf}.${process.pid}.${Date.now()}.tmp`;
await pipeline(source.stream, fs.createWriteStream(tmp));
if (opts.useCache === false) return tmp;
fs.renameSync(tmp, cf);
return cf;
}
async function persist(key, localPath) {
const k = normalizeKey(key);
if (!k) return false;
const src = path.resolve(localPath);
if (!fs.existsSync(src)) return false;
if (!remote) return true;
const info = await head(k);
if (info && info.size === fs.statSync(src).size) {
if (!keepLocal) unlinkLocalOnly(k);
return true;
}
await putFile(k, src);
if (!keepLocal) {
try { fs.unlinkSync(src); } catch {}
const cf = cacheFile(k);
if (cf && fs.existsSync(cf)) {
try { fs.unlinkSync(cf); } catch {}
}
}
return true;
}
async function streamTo(res, key, opts = {}) {
const k = normalizeKey(key);
if (!k) return false;
if (localFallback && localExists(k)) {
const fp = localFile(k);
if (opts.cacheControl) res.setHeader('Cache-Control', opts.cacheControl);
if (opts.download) {
res.download(fp, opts.name || path.basename(fp), opts.callback || (() => {}));
return true;
}
res.setHeader('Content-Type', opts.contentType || mimeFor(k));
res.sendFile(fp);
return true;
}
const source = await getStream(k);
if (!source) return false;
if (opts.cacheControl) res.setHeader('Cache-Control', opts.cacheControl);
res.setHeader('Content-Type', opts.contentType || source.contentType || mimeFor(k));
if (source.contentLength) res.setHeader('Content-Length', String(source.contentLength));
if (opts.download) {
const name = opts.name || path.basename(k);
const ascii = name.replace(/[^\x20-\x7E]/g, '_').replace(/"/g, '');
res.setHeader('Content-Disposition', `attachment; filename="${ascii}"; filename*=UTF-8''${encodeURIComponent(name)}`);
}
await pipeline(source.stream, res);
return true;
}
async function downloadAll(destDir, opts = {}) {
fs.mkdirSync(destDir, { recursive: true });
const root = path.resolve(destDir);
const objects = await listAll('');
let count = 0;
for (const obj of objects) {
if (!isObjectKey(obj.key)) continue;
const k = normalizeKey(obj.key);
if (!k) continue;
const target = path.resolve(root, ...k.split('/'));
if (!target.startsWith(root + path.sep)) continue;
const source = await getStream(k);
if (!source) continue;
fs.mkdirSync(path.dirname(target), { recursive: true });
await pipeline(source.stream, fs.createWriteStream(target));
count++;
}
if (opts.pruneThumbs !== false) {
const thumbs = path.join(root, '.thumbs');
if (fs.existsSync(thumbs)) fs.rmSync(thumbs, { recursive: true, force: true });
}
return count;
}
async function uploadTree(srcDir) {
const root = path.resolve(srcDir);
if (!fs.existsSync(root)) return 0;
let count = 0;
const walk = async (abs, rel) => {
for (const name of fs.readdirSync(abs)) {
const childAbs = path.join(abs, name);
const childRel = rel ? `${rel}/${name}` : name;
let st;
try {
st = fs.statSync(childAbs);
} catch {
continue;
}
if (st.isDirectory()) {
if (name === '.thumbs' || name === '.cache') continue;
await walk(childAbs, childRel);
continue;
}
const k = normalizeKey(childRel);
if (!k) continue;
await putFile(k, childAbs);
count++;
}
};
await walk(root, '');
return count;
}
async function ensureBucket() {
if (!remote || !bucket) return false;
try {
await client.send(new HeadBucketCommand({ Bucket: bucket }));
return true;
} catch (err) {
const status = err && err.$metadata && err.$metadata.httpStatusCode;
const name = err && (err.name || err.Code);
if (status && status !== 404 && name !== 'NotFound' && name !== 'NoSuchBucket') throw err;
}
try {
await client.send(new CreateBucketCommand({ Bucket: bucket }));
return true;
} catch (err) {
const name = err && (err.name || err.Code);
if (name === 'BucketAlreadyOwnedByYou' || name === 'BucketAlreadyExists') return true;
throw err;
}
}
async function usage() {
const objects = (await listAll('')).filter(o => isObjectKey(o.key));
return {
count: objects.length,
size_bytes: objects.reduce((sum, o) => sum + (o.size || 0), 0),
bucket: remote ? bucket : null,
prefix: prefix || null,
};
}
function pruneCache(now = Date.now()) {
if (!cacheMaxAgeMs || !fs.existsSync(cacheDir)) return 0;
let removed = 0;
const walk = (abs) => {
for (const name of fs.readdirSync(abs)) {
const childAbs = path.join(abs, name);
let st;
try {
st = fs.statSync(childAbs);
} catch {
continue;
}
if (st.isDirectory()) {
walk(childAbs);
continue;
}
if (now - st.mtimeMs > cacheMaxAgeMs) {
try { fs.unlinkSync(childAbs); removed++; } catch {}
}
}
};
walk(cacheDir);
return removed;
}
return {
bucket,
dir,
cacheDir,
isRemote,
keyFromPath,
publicPath,
localFile,
localExists,
unlinkLocalOnly,
normalizeKey,
put,
putFile,
head,
exists,
sizeOf,
getStream,
getBuffer,
del,
copyObject,
listAll,
localize,
persist,
streamTo,
downloadAll,
uploadTree,
ensureBucket,
usage,
pruneCache,
};
}
module.exports = { createStorage, mimeFor, normalizeKey };
+15 -32
View File
@@ -9,7 +9,7 @@ const crypto = require('crypto');
const PHOTO_MAX_ATTEMPTS = 3; const PHOTO_MAX_ATTEMPTS = 3;
const PHOTO_AI_TIMEOUT_MS = 300000; const PHOTO_AI_TIMEOUT_MS = 300000;
function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntries, sharp, photoAiUrl, uploadsDir, originalsDir }) { function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntries, sharp, photoAiUrl, uploadsDir, originalsDir, storage }) {
const AI_URL = photoAiUrl || process.env.PHOTO_AI_URL || ''; const AI_URL = photoAiUrl || process.env.PHOTO_AI_URL || '';
const IDLE_MIN = 2000; const IDLE_MIN = 2000;
const IDLE_MAX = 30000; const IDLE_MAX = 30000;
@@ -71,30 +71,7 @@ function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntrie
} }
} }
function safeUnlinkPath(p) { async function enhanceWithSharp(srcKey, params) {
try {
if (!p) return;
const abs = path.resolve(p);
const root = path.resolve(uploadsDir);
if (abs !== root && abs.startsWith(root + path.sep)) fs.unlinkSync(abs);
} catch (e) { console.error('photo worker unlink:', e.message); }
}
function backupOldFile(oldRelPath) {
if (!oldRelPath) return null;
const oldAbs = path.join(uploadsDir, String(oldRelPath).replace(/^\/+/, '').replace(/^uploads\//, ''));
if (!fs.existsSync(oldAbs)) return null;
const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg');
const backupPath = path.join(originalsDir, backupName);
try {
fs.renameSync(oldAbs, backupPath);
return `/uploads/.originals/${backupName}`;
} catch (e) {
console.error('photo worker backup failed:', e.message);
return null;
}
}
async function enhanceWithSharp(srcAbs, params) {
if (!sharp) throw new Error('sharp недоступен на сервере'); if (!sharp) throw new Error('sharp недоступен на сервере');
const p = params || {}; const p = params || {};
const clamp = (v, min, max, def) => { const clamp = (v, min, max, def) => {
@@ -106,7 +83,9 @@ function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntrie
const saturate = clamp(p.saturate, 0, 300, 100); const saturate = clamp(p.saturate, 0, 300, 100);
const sharpAmt = clamp(p.sharp, 0, 100, 0); const sharpAmt = clamp(p.sharp, 0, 100, 0);
const denoise = clamp(p.denoise, 0, 100, 0); const denoise = clamp(p.denoise, 0, 100, 0);
let pipeline = sharp(srcAbs).rotate(); const srcPath = await storage.localize(srcKey);
if (!srcPath) throw new Error('Файл фото не найден');
let pipeline = sharp(srcPath).rotate();
if (denoise > 0) pipeline = pipeline.median(denoise > 70 ? 5 : 3); if (denoise > 0) pipeline = pipeline.median(denoise > 70 ? 5 : 3);
pipeline = pipeline.modulate({ brightness: brightness / 100, saturation: saturate / 100 }); pipeline = pipeline.modulate({ brightness: brightness / 100, saturation: saturate / 100 });
if (contrast !== 100) { if (contrast !== 100) {
@@ -117,13 +96,16 @@ function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntrie
pipeline = pipeline.sharpen({ sigma: 0.5 + (sharpAmt / 100) * 1.5, m1: 0, m2: 1 + sharpAmt / 50 }); pipeline = pipeline.sharpen({ sigma: 0.5 + (sharpAmt / 100) * 1.5, m1: 0, m2: 1 + sharpAmt / 50 });
} }
const newName = crypto.randomBytes(12).toString('hex') + '.jpg'; const newName = crypto.randomBytes(12).toString('hex') + '.jpg';
await pipeline.jpeg({ quality: 92, mozjpeg: true }).toFile(path.join(uploadsDir, newName)); const outPath = path.join(uploadsDir, newName);
await pipeline.jpeg({ quality: 92, mozjpeg: true }).toFile(outPath);
await storage.persist(newName, outPath);
return `/uploads/${newName}`; return `/uploads/${newName}`;
} }
async function runAiEnhance(srcAbs) { async function runAiEnhance(srcKey) {
if (!AI_URL) throw new Error('PHOTO_AI_URL не настроен'); if (!AI_URL) throw new Error('PHOTO_AI_URL не настроен');
const buf = fs.readFileSync(srcAbs); const buf = await storage.getBuffer(srcKey);
if (!buf) throw new Error('Файл фото не найден');
const fd = new FormData(); const fd = new FormData();
fd.append('image', new Blob([buf], { type: 'image/jpeg' }), 'photo.jpg'); fd.append('image', new Blob([buf], { type: 'image/jpeg' }), 'photo.jpg');
fd.append('scale', '2'); fd.append('scale', '2');
@@ -135,7 +117,9 @@ function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntrie
if (!resp.ok) throw new Error('AI service error: ' + resp.status); if (!resp.ok) throw new Error('AI service error: ' + resp.status);
const out = Buffer.from(await resp.arrayBuffer()); const out = Buffer.from(await resp.arrayBuffer());
const newName = crypto.randomBytes(12).toString('hex') + '.jpg'; const newName = crypto.randomBytes(12).toString('hex') + '.jpg';
fs.writeFileSync(path.join(uploadsDir, newName), out); const outPath = path.join(uploadsDir, newName);
fs.writeFileSync(outPath, out);
await storage.persist(newName, outPath);
return `/uploads/${newName}`; return `/uploads/${newName}`;
} }
async function applyResult(job, newPath) { async function applyResult(job, newPath) {
@@ -159,9 +143,8 @@ function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntrie
return true; return true;
} }
const photoPath = rows[0].photo_path; const photoPath = rows[0].photo_path;
const srcAbs = path.join(uploadsDir, String(photoPath).replace(/^\/+/, '').replace(/^uploads\//, ''));
try { try {
const newPath = job.action === 'ai' ? await runAiEnhance(srcAbs) : await enhanceWithSharp(srcAbs, job.params); const newPath = job.action === 'ai' ? await runAiEnhance(photoPath) : await enhanceWithSharp(photoPath, job.params);
await applyResult(job, newPath); await applyResult(job, newPath);
stats.done++; stats.done++;
stats.last_at = new Date().toISOString(); stats.last_at = new Date().toISOString();