diff --git a/.env.example b/.env.example index ed645dd..22e5cde 100644 --- a/.env.example +++ b/.env.example @@ -31,3 +31,29 @@ WG_HANDSHAKE_TIMEOUT=60 # ИИ-улучшение фото (Real-ESRGAN), пусто = контейнер photo-ai PHOTO_AI_URL= PHOTO_AI_MAX_PIXELS=4000000 + +# === Хранилище файлов (S3: SeaweedFS по умолчанию / MinIO) === +# local — файлы в ./uploads (по умолчанию), s3 — объекты в бакете S3/MinIO. +STORAGE_DRIVER=local +# Читать локальную копию, если объект ещё не перенесён в S3 (1 — включено). +STORAGE_LOCAL_FALLBACK=1 +# Оставлять локальную копию после выгрузки в S3 (1 — оставлять, 0 — удалять). +STORAGE_KEEP_LOCAL=0 +# Сколько часов хранить локальный кэш оригиналов (для sharp/миниатюр), 0 — не чистить. +STORAGE_CACHE_MAX_AGE_HOURS=168 +# Образ S3-сервиса. По умолчанию SeaweedFS (свободный S3-сервер). +# Для MinIO: S3_IMAGE=minio/minio:<тег> и запуск через docker-compose.minio.yml. +S3_IMAGE=chrislusf/seaweedfs:latest +S3_ENDPOINT=http://s3:9000 +S3_REGION=us-east-1 +S3_BUCKET=whatido +# Логин/пароль S3 (для SeaweedFS — AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY, +# для MinIO — MINIO_ROOT_USER/MINIO_ROOT_PASSWORD). +S3_ACCESS_KEY=whatido +S3_SECRET_KEY=замените-на-длинный-секрет +# Path-style адресация (1 — включено, нужно для MinIO и SeaweedFS). +S3_FORCE_PATH_STYLE=1 +# Необязательный префикс ключей внутри бакета (например, prod). +S3_PREFIX= +# Показывать веб-консоль MinIO (on/off), только для docker-compose.minio.yml +MINIO_BROWSER=off diff --git a/AGENTS.md b/AGENTS.md index 02f3755..0464ea8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,9 +8,9 @@ This document defines how AI agents should work with the WhatIDo codebase. Follo **WhatIDo** — Accounting system for an educational center: attendance journal, student project works, group gallery, detached files, and public showcase pages (share links). -- **Stack**: Node.js 20 + Express, PostgreSQL 16, Docker Compose, Tailscale (Serve/Funnel) -- **Architecture**: Single Express server (`server.js`) + static frontend in `public/` -- **Deployment**: Docker Compose (app + db + tailscale), bind-mounted uploads, named volume for Postgres data +- **Stack**: Node.js 20 + Express, PostgreSQL 16, Docker Compose, S3-совместимое хранилище файлов, Tailscale (Serve/Funnel) +- **Architecture**: Single Express server (`server.js`) + storage abstraction (`storage.js`) + static frontend in `public/` +- **Deployment**: Docker Compose (app + db + s3 + tailscale), bind-mounted uploads, named volumes for Postgres and S3 data - **Auth**: Admin-only via `X-Admin-Token` header (value = `ADMIN_PASSWORD` env var). No user sessions. --- @@ -34,9 +34,18 @@ This document defines how AI agents should work with the WhatIDo codebase. Follo ### 3. File Uploads - **Multer configs**: `upload` (images only), `adminUpload` (wider allowed ext), `uploadBackup` (restore) - **Limits**: 10 MB/file, 30 MB total per entry -- **Storage**: `uploads/` bind-mounted to host, filenames = `timestamp-random.ext` +- **Staging**: Multer always writes to `uploads/` (`timestamp-random.ext`); a global `res.on('finish')` hook persists each uploaded file through `storage.persist` on successful responses (only when `STORAGE_DRIVER=s3`) - **HEIC**: Auto-converted to JPEG via `heic-convert` -- **Cleanup**: `safeUnlink` / `sweepOrphanedUploads` — never delete outside `uploads/` +- **Cleanup**: `safeUnlink` / `sweepOrphanedUploads` — never delete outside `uploads/` or the configured bucket + +### 3a. Storage (`storage.js`) +- **Drivers**: `local` (default, files in `uploads/`) and `s3` (S3-compatible: SeaweedFS by default, MinIO via `docker-compose.minio.yml`) +- **Keys are stable**: DB stores `/uploads/`; S3 object keys are the same `` (plus `.originals/`). Never change key format — it would break existing DB rows and URLs +- **API**: `put`, `putFile`, `head`, `exists`, `sizeOf`, `getStream`, `getBuffer`, `del`, `copyObject`, `listAll`, `localize`, `persist`, `streamTo`, `downloadAll`, `uploadTree`, `ensureBucket`, `usage`, `pruneCache` +- **Rules**: never call `fs.*` on `uploads/` directly in request/worker code — use `storage.*`. `safeUnlink` is the only deletion helper (local + remote, idempotent) +- **Read path**: `STORAGE_LOCAL_FALLBACK=1` prefers a local file when it still exists (covers in-flight uploads and partial migration); otherwise the app streams the object from S3 +- **Cache**: `.thumbs` (WebP miniatures) and `.cache` (originals localized for sharp/zip) live inside `uploads/` and are pruned hourly (`STORAGE_CACHE_MAX_AGE_HOURS`) +- **Never publish the S3 API port**: only `127.0.0.1` on the host, file access stays behind app auth/rate limits ### 4. API Patterns - **Admin routes**: `requireAdmin` middleware (checks `X-Admin-Token`) @@ -102,6 +111,15 @@ This document defines how AI agents should work with the WhatIDo codebase. Follo - Update Multer `fileFilter` functions - Keep `MAX_TOTAL_UPLOAD_BYTES` and per-file limit in sync +### Migrate files to S3 / switch storage driver +1. `docker compose up -d s3` +2. `docker compose exec -T app node scripts/migrate-to-s3.js --dry-run` then without the flag (idempotent, size-checked, keeps local files) +3. `docker compose exec -T app node scripts/migrate-to-s3.js --verify-only` +4. Set `STORAGE_DRIVER=s3` in `.env`, `docker compose up -d app` +5. After verification: `docker compose exec -T app node scripts/migrate-to-s3.js --delete-local` +- Rollback: `STORAGE_DRIVER=local` + `docker compose up -d app` +- Do not run `--delete-local` before the app serves reads from S3 and the verification passes + --- ## Testing & Verification @@ -121,6 +139,11 @@ curl -H "X-Admin-Token: $ADMIN_PASSWORD" http://localhost:3003/api/groups # Run backup/restore scripts ./scripts/backup.sh ./scripts/restore.sh backups/whatido-backup-.tar.gz + +# Storage checks +docker compose up -d s3 +docker compose exec -T app node scripts/migrate-to-s3.js --dry-run +docker compose exec -T app node scripts/migrate-to-s3.js --verify-only ``` --- @@ -128,6 +151,7 @@ curl -H "X-Admin-Token: $ADMIN_PASSWORD" http://localhost:3003/api/groups ## Security Checklist (before any change) - [ ] No SQL interpolation — only `$1`, `$2`... - [ ] Upload path validation via `isSafeUploadPath` / `safeUnlink` +- [ ] File I/O через `storage.*`, ключи объектов не выходят за пределы бакета/`uploads/` - [ ] Rate limiter on new public routes - [ ] Admin routes behind `requireAdmin` - [ ] No secrets in code — only via env vars @@ -141,15 +165,19 @@ curl -H "X-Admin-Token: $ADMIN_PASSWORD" http://localhost:3003/api/groups | File | Purpose | |------|---------| | `server.js` | Entire backend (Express, routes, DB, uploads, backup) | -| `worker.js` | Background AI auto-check worker for entry messages | +| `storage.js` | Storage abstraction: `local` and `s3` drivers, key normalization, cache/thumb helpers | +| `worker.js` | Background AI auto-check worker for entry messages + photo enhance worker | | `db/init.sql` | Initial schema (runs on fresh DB) | | `db/migration.sql` | Idempotent migrations for existing DBs | -| `docker-compose.yml` | Service definitions (app, db, tailscale) | +| `docker-compose.yml` | Service definitions (app, db, s3, tailscale) | +| `docker-compose.minio.yml` | Override: S3 service backed by MinIO instead of SeaweedFS | | `Dockerfile` | App image build | | `public/*.html` | Frontend pages | | `public/admin.js` | Shared frontend logic | -| `scripts/backup.sh` | Host-level backup script | -| `scripts/restore.sh` | Host-level restore script | +| `scripts/backup.sh` | Host-level backup script (DB dump + storage export) | +| `scripts/restore.sh` | Host-level restore script (DB dump + storage import) | +| `scripts/storage-sync.js` | Export/import all storage objects (used by backup/restore) | +| `scripts/migrate-to-s3.js` | One-off/idempotent migration `uploads/` -> S3 bucket | | `scripts/deploy.sh` | Deploy script (pull master, build image with commit version, restart app) | | `start-tailscale.sh` | Tailscale container entrypoint | | `.env.example` | Env var template | @@ -160,6 +188,9 @@ curl -H "X-Admin-Token: $ADMIN_PASSWORD" http://localhost:3003/api/groups - ❌ Add dependencies without updating `package.json` and rebuilding - ❌ Write files outside `uploads/` or `certs/` +- ❌ Touch `uploads/` with `fs.*` in request/worker code — use `storage.*` (files may live only in S3) +- ❌ Run `migrate-to-s3.js --delete-local` before verification and cutover +- ❌ Expose the S3 API port publicly (only `127.0.0.1` in compose) - ❌ Commit `.env`, `certs/`, `uploads/`, `backups/`, `node_modules/` - ❌ Expose DB port (5432) outside docker network - ❌ Use `eval`, `Function` constructor, or dynamic code execution @@ -182,6 +213,11 @@ docker compose logs -f app # DB shell docker compose exec db psql -U app -d whereldo +# S3 storage status and migration verification +docker compose up -d s3 +docker compose exec -T app node scripts/migrate-to-s3.js --verify-only +docker compose exec -T app node scripts/migrate-to-s3.js --delete-local + # Tailscale status docker exec -it whatido-tailscale-1 tailscale status diff --git a/README.md b/README.md index df71ba5..72d3bef 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ - **Share-ссылки** — публичные страницы-витрины с выбором группы / воспитанника / диапазона дат - **Дашборд** — статистика, активные группы, активность за 14 дней, последние записи, топ воспитанников - **Резервное копирование** — экспорт/импорт полного дампа (БД + файлы) в `tar.gz` +- **Хранилище файлов** — локальный каталог `uploads/` или S3-совместимый сервис (`s3`: SeaweedFS, либо MinIO через оверрайд), перенос файлов скриптом миграции - **Настройки** — тексты футера, анти-спам интервал - **Публикация через Tailscale** — приложение открывается по постоянному адресу `https://whatido..ts.net` без проброса портов, внешнего IP и reverse-proxy @@ -20,6 +21,7 @@ - Node.js + Express - PostgreSQL (pg) - Multer (загрузка файлов), Tar (бэкапы) +- S3-совместимое хранилище (AWS SDK v3): сервис `s3` (SeaweedFS / MinIO) - Lucide (иконки UI) - Docker / Docker Compose - Tailscale (Serve / Funnel) — публикация по HTTPS @@ -244,7 +246,71 @@ docker compose exec cloudflared wg show # есть handshake — VPN подн ## Хранилище файлов -Загруженные фото и файлы хранятся в каталоге `uploads/` на хосте и монтируются в контейнер (`./uploads:/app/uploads`). Это даёт прямой доступ к данным из-под хост-системы. Данные БД хранятся в именованном томе `pgdata`. +По умолчанию загруженные фото и файлы хранятся в каталоге `uploads/` на хосте и монтируются в контейнер (`./uploads:/app/uploads`) — это драйвер `local`. Данные БД хранятся в именованном томе `pgdata`. + +Дополнительно поддерживается **S3-совместимое хранилище** (сервис `s3` в compose, драйвер `s3`). Все обращения к файлам идут через приложение: URL (`/uploads/...`, `/uploads/thumb/...`, `/api/files/:token`, share-ссылки) и записи в БД (`/uploads/<файл>`) не меняются, поэтому переключение драйвера не требует миграции данных в БД. + +### Сервис `s3` + +```bash +docker compose up -d s3 # поднимает S3-хранилище (том s3-data) +``` + +- **По умолчанию — SeaweedFS** (`chrislusf/seaweedfs`): свободный S3-сервер; API слушает `127.0.0.1:9000` на хосте и `s3:9000` внутри compose-сети. +- **MinIO**: официальные свободные образы `minio/minio` удалены из Docker Hub, поэтому MinIO подключается через оверрайд и образ из доступного вам зеркала: + +```bash +S3_IMAGE=<ваш-образ-minio> docker compose -f docker-compose.yml -f docker-compose.minio.yml up -d s3 +``` + +Бакет создаётся автоматически при старте приложения (`ensureBucket`) или скриптом миграции. Анонимный доступ к API хранилища закрыт: порт `9000` не публикуется наружу (только loopback), доступ к файлам остаётся через приложение с его аутентификацией и rate limit. + +### Переменные окружения + +| Переменная | По умолчанию | Назначение | +|---|---|---| +| `STORAGE_DRIVER` | `local` | `local` — файлы в `uploads/`, `s3` — объекты в бакете | +| `S3_ENDPOINT` | `http://s3:9000` | Адрес S3 API внутри compose-сети | +| `S3_BUCKET` | `whatido` | Бакет для объектов | +| `S3_ACCESS_KEY` / `S3_SECRET_KEY` | `whatido` / — | Доступ к хранилищу (для MinIO это root-пользователь) | +| `S3_FORCE_PATH_STYLE` | `1` | Path-style адресация (нужна MinIO/SeaweedFS) | +| `S3_PREFIX` | — | Необязательный префикс ключей внутри бакета | +| `STORAGE_LOCAL_FALLBACK` | `1` | Читать локальный файл, если объекта в S3 ещё нет | +| `STORAGE_KEEP_LOCAL` | `0` | Оставлять локальную копию после выгрузки в S3 | +| `STORAGE_CACHE_MAX_AGE_HOURS` | `168` | Срок жизни локального кэша оригиналов (для sharp/миниатюр) | + +### Переход на S3 (миграция) + +Порядок не прерывает работу: файлы сначала копируются в бакет, локальные остаются на месте и продолжают использоваться. + +```bash +# 1) поднять хранилище +docker compose up -d s3 + +# 2) предпросмотр и загрузка файлов в бакет (идемпотентно, по размеру объекта) +docker compose exec -T app node scripts/migrate-to-s3.js --dry-run +docker compose exec -T app node scripts/migrate-to-s3.js + +# 3) проверить, что все объекты на месте (ничего не меняет) +docker compose exec -T app node scripts/migrate-to-s3.js --verify-only +``` + +Дальше включить драйвер `s3` и перезапустить приложение: + +```bash +# в .env: STORAGE_DRIVER=s3 +docker compose up -d app +``` + +Новые загрузки уходят в бакет (локальная копия удаляется, если `STORAGE_KEEP_LOCAL=0`), старые файлы ещё читаются из `uploads/` благодаря `STORAGE_LOCAL_FALLBACK=1`. Когда всё проверено — удалите локальные копии: + +```bash +docker compose exec -T app node scripts/migrate-to-s3.js --delete-local +``` + +Откат в любой момент: `STORAGE_DRIVER=local` + `docker compose up -d app` (пока локальные копии не удалены). + +Объём и состав хранилища видны в админке: Настройки → Системная информация (блок «Хранилище»). ## Бэкапы @@ -264,6 +330,8 @@ docker compose exec cloudflared wg show # есть handshake — VPN подн Форматы не взаимозаменяемы: скриптовый архив содержит `db.sql.gz` + `_uploads/` (перенос на другой хост через `scripts/restore.sh`), а веб-архив из админки — `data.json` + `uploads/` (кнопка «Восстановить»). Если в админку загрузить скриптовый архив, сервер вернёт подсказку, какой инструмент использовать. +Файлы попадают в бэкап из активного хранилища: при `STORAGE_DRIVER=s3` админ-бэкап и `scripts/backup.sh` выгружают объекты из бакета (`scripts/storage-sync.js export`), а восстановление загружает их обратно (`scripts/storage-sync.js import`). Миниатюры (`.thumbs`) в архив не включаются — они пересоздаются по запросу. + ## Безопасность - **Пароль администратора** обязателен (`ADMIN_PASSWORD`); фолбэка на `admin` нет. @@ -299,17 +367,19 @@ docker compose exec cloudflared wg show # есть handshake — VPN подн ## Структура проекта ``` -├── docker-compose.yml # сервисы: app + db + tailscale +├── docker-compose.yml # сервисы: app + db + s3 (+ опционально tailscale) +├── docker-compose.minio.yml # оверрайд: S3-сервис на MinIO вместо SeaweedFS ├── .env.example # шаблон переменных окружения ├── Dockerfile # сборка образа (Node 20, генерация TLS-сертификата) ├── server.js # Express-приложение -├── worker.js # фоновый worker AI-проверки записей +├── storage.js # абстракция хранилища: драйверы local и s3 +├── worker.js # фоновый worker AI-проверки и ИИ-улучшения фото ├── certs/ # cert.pem приложения (монтируется в tailscale, в git не хранится) ├── db/ │ ├── init.sql # схема при первом запуске │ └── migration.sql # миграции существующей БД ├── public/ # статика (HTML/CSS/JS админки и витрин) -├── scripts/ # вспомогательные скрипты -├── uploads/ # загруженные файлы (bind-монт, вне git) +├── scripts/ # вспомогательные скрипты (backup/restore/deploy, migrate-to-s3, storage-sync) +├── uploads/ # локальные файлы и кэш миниатюр (bind-монт, вне git) └── backups/ # локальные бэкапы ``` \ No newline at end of file diff --git a/docker-compose.minio.yml b/docker-compose.minio.yml new file mode 100644 index 0000000..9cb51ba --- /dev/null +++ b/docker-compose.minio.yml @@ -0,0 +1,26 @@ +# Переопределение S3-сервиса на MinIO. +# Использование: +# S3_IMAGE=minio/minio:RELEASE.2025-04-22T22-12-26Z \ +# docker compose -f docker-compose.yml -f docker-compose.minio.yml up -d s3 +# +# Учтите: MinIO прекратил публикацию свободных образов (docker.io/minio/minio +# удалён), поэтому образ нужно взять из доступного вам зеркала/архива и указать +# его в S3_IMAGE. +services: + s3: + image: ${S3_IMAGE:-minio/minio:latest} + container_name: whatido-s3 + restart: unless-stopped + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${S3_ACCESS_KEY:-whatido} + MINIO_ROOT_PASSWORD: ${S3_SECRET_KEY:-whatido-secret} + MINIO_BROWSER: ${MINIO_BROWSER:-off} + TZ: Europe/Moscow + expose: + - "9000" + ports: + - "127.0.0.1:9000:9000" + - "127.0.0.1:9001:9001" + volumes: + - s3-data:/data diff --git a/docker-compose.yml b/docker-compose.yml index d52a216..e844ad2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,7 @@ name: whatido services: db: image: postgres:16-alpine + restart: unless-stopped environment: POSTGRES_DB: whereldo POSTGRES_USER: app @@ -24,6 +25,7 @@ services: args: GIT_COMMIT: ${GIT_COMMIT:-} GIT_COMMIT_DATE: ${GIT_COMMIT_DATE:-} + restart: unless-stopped extra_hosts: - "host.docker.internal:host-gateway" expose: @@ -43,12 +45,46 @@ services: PHOTO_AI_URL: ${PHOTO_AI_URL:-http://photo-ai:8080} NODE_ENV: production TZ: Europe/Moscow + STORAGE_DRIVER: ${STORAGE_DRIVER:-local} + STORAGE_LOCAL_FALLBACK: ${STORAGE_LOCAL_FALLBACK:-1} + STORAGE_KEEP_LOCAL: ${STORAGE_KEEP_LOCAL:-0} + STORAGE_CACHE_MAX_AGE_HOURS: ${STORAGE_CACHE_MAX_AGE_HOURS:-168} + S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-whatido} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-whatido} + S3_SECRET_KEY: ${S3_SECRET_KEY:-whatido-secret} + S3_FORCE_PATH_STYLE: ${S3_FORCE_PATH_STYLE:-1} + S3_PREFIX: ${S3_PREFIX:-} depends_on: db: condition: service_healthy volumes: - ./uploads:/app/uploads + # S3-совместимое хранилище файлов. API 9000 доступен только внутри сети compose + # (плюс loopback хоста для отладки/миграции). + # По умолчанию — SeaweedFS: свободный S3-сервер, доступный в Docker Hub. + # Для MinIO (если образ доступен в вашем зеркале) используйте: + # docker compose -f docker-compose.yml -f docker-compose.minio.yml up -d s3 + # Перенос файлов из ./uploads в бакет: + # docker compose exec -T app node scripts/migrate-to-s3.js --dry-run + s3: + image: ${S3_IMAGE:-chrislusf/seaweedfs:latest} + container_name: whatido-s3 + restart: unless-stopped + command: server -dir=/data -s3 -s3.port=9000 + environment: + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-whatido} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY:-whatido-secret} + TZ: Europe/Moscow + expose: + - "9000" + ports: + - "127.0.0.1:9000:9000" + volumes: + - s3-data:/data + # Публикация через Tailscale (Serve / Funnel) без проброса портов. # Приложение доступно по https://whatido..ts.net # tailscale: @@ -134,3 +170,4 @@ services: volumes: pgdata: photo-ai-models: + s3-data: diff --git a/package-lock.json b/package-lock.json index bc54721..fdf2f6e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "whereldo", "version": "1.0.0", "dependencies": { + "@aws-sdk/client-s3": "^3.1141.0", "bcrypt": "^5.1.1", "express": "^4.21.0", "express-rate-limit": "^8.7.0", @@ -20,6 +21,314 @@ "tar": "^7.4.3" } }, + "node_modules/@aws-sdk/checksums": { + "version": "3.1001.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1001.1.tgz", + "integrity": "sha512-x12Q17KYlJAd3nKf8LV5LV0vt8sh8/6YfQLGPtrGnQf/tW4jqxPGq5GPpuVitpQYM3eUR4XB7CbxZf751NMbLw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.1141.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1141.0.tgz", + "integrity": "sha512-uOVH37xGLenAdJkCPCin/JJG2PgWrFcSsDnQ9+C9Zq8N9Oalo5ol4xmn5fG28iWAlA/b/9boQZgHbMh+UsIhcg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/checksums": "^3.1001.1", + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/credential-provider-node": "^3.972.84", + "@aws-sdk/middleware-sdk-s3": "^3.972.77", + "@aws-sdk/signature-v4-multi-region": "^3.996.47", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.978.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.978.1.tgz", + "integrity": "sha512-LbY9aGsEiznDWmUc30Nwv3aIX/+dbwTx8KfS0yOC3NPYMO+O91e6jkT1azf34FwjOndq8/Q+RcVVZz5xnerwdg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.6", + "@aws-sdk/xml-builder": "^3.972.41", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.35.0", + "@smithy/signature-v4": "^5.7.3", + "@smithy/types": "^4.19.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.72", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.72.tgz", + "integrity": "sha512-xTKO/FWJPozTIXbozVnVGoNBhaGba8TBcx+KyUjRVeOlXE+dUc7GTR1cLvu0uTdIdmemzaFbqqCshXeZA1fZew==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.74", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.74.tgz", + "integrity": "sha512-u91E/hT8f4d1xy0Jl7VG4nVKJ3lxbrZkoBTeSVoJdWBiSEUMwMS/9+e0H/aJVQV//Lt5wuzP+E69v4aRSsNTmw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.17", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.17.tgz", + "integrity": "sha512-ged4KXdBkvIC81bLvNHHuQKdKak/VXhQTR1NWYTTqW0474nlmsxy9O/vlgTIohDDWH3xpBdtVMZRyjb+DnocDA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/credential-provider-env": "^3.972.72", + "@aws-sdk/credential-provider-http": "^3.972.74", + "@aws-sdk/credential-provider-login": "^3.972.79", + "@aws-sdk/credential-provider-process": "^3.972.72", + "@aws-sdk/credential-provider-sso": "^3.973.16", + "@aws-sdk/credential-provider-web-identity": "^3.972.78", + "@aws-sdk/nested-clients": "^3.997.46", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/credential-provider-imds": "^4.5.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.79", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.79.tgz", + "integrity": "sha512-L+Z85anONJd8MaiuraO4wRxATCdEejBZ3K3eymzWI5JPXa9sOS9CkIm72PBKqXKX+Z9p9NGMX5AIMXm0LEflgw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/nested-clients": "^3.997.46", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.84", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.84.tgz", + "integrity": "sha512-oHt854odINVwzwsh+c5x69j0ajm4DbqqqVJ+O1ECsCIZeMDAbzFpXItaqP7UZstJj/ATdTk/KFSH0LaNAgV+kA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.72", + "@aws-sdk/credential-provider-http": "^3.972.74", + "@aws-sdk/credential-provider-ini": "^3.973.17", + "@aws-sdk/credential-provider-process": "^3.972.72", + "@aws-sdk/credential-provider-sso": "^3.973.16", + "@aws-sdk/credential-provider-web-identity": "^3.972.78", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/credential-provider-imds": "^4.5.2", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.72", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.72.tgz", + "integrity": "sha512-rLIp2xbMjX/k9/od7APpqq1ZgXXnV0pOL1Th3ZsL8Wu0TRtBsDTVS8iPqcfRFcHakFxPvR04OSTv2ka2qOb/2A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.16", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.16.tgz", + "integrity": "sha512-IGihaJfFZYacJJr/odqILCoK7W/mvrZ7cuK7ECn3sAu4vLC6u0V8bS7mCGbdugJ8Aum2tnvqmx0F2MRFp2rn9g==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/nested-clients": "^3.997.46", + "@aws-sdk/token-providers": "3.1138.0", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.78", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.78.tgz", + "integrity": "sha512-/y9WvNtlcPBGLR0qc1a+9J/xtYZfVczvLUOuXaVWylzttH7ewsxwHtjmiJSolNrVSDorIxHGHMU61CbonRkmwA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/nested-clients": "^3.997.46", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.972.77", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.77.tgz", + "integrity": "sha512-E7W2UOeUoc+lg3uIfR/dM7ZwusHwhBQrKMnlkRv4EXRR+C0YtV1pg25xC7GdZIhXH+NAMgZPCbE7o5to2cjFiw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/signature-v4-multi-region": "^3.996.47", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.46", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.46.tgz", + "integrity": "sha512-oRxtBcka/JGHGs9l9p9IVajGoTP8vTPmoAzdHGy4Qcy9P5vPnDf6nhIeM/COQNY9k/OahImTRaLkHftoXvfcmQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/signature-v4-multi-region": "^3.996.47", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/fetch-http-handler": "^5.8.0", + "@smithy/node-http-handler": "^4.12.1", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.47", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.47.tgz", + "integrity": "sha512-Zk08macMvQTHzQJCLJVkOlviVoqwYMrpXv4lmLN7b7sAbiMoOK7Go0NYdR5UeF+MW8LIbRmwrNy9u/5VvX1U5g==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.6", + "@smithy/signature-v4": "^5.7.3", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1138.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1138.0.tgz", + "integrity": "sha512-GpyAr0DD63YOEmYFM6Df+gJuIgC92MMTiBK4FTKfxii5MJ9ge20epR7LyroulscYlG89J+ZB2ivFDPjvfQhzdw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.978.1", + "@aws-sdk/nested-clients": "^3.997.46", + "@aws-sdk/types": "^3.974.6", + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.6", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.6.tgz", + "integrity": "sha512-v/clNZzZnDxGyvpHMOGpJKVXFAExJzUNAAjaWGdcx8QAcXLGwTaOkw33p5SHAi0YAioK32xB3hWwOekRVfmfKg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.41.tgz", + "integrity": "sha512-ctjVSyCMegrWfXlx6VqzSBFI6UqmQ5ZlnfMhdLIiWmhoH8UAQxSCP5N3OpG7X3k4LnS7ou74C4mt20+bfTW2aQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@emnapi/runtime": { "version": "1.11.3", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", @@ -654,6 +963,87 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "license": "ISC" }, + "node_modules/@smithy/core": { + "version": "3.35.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.35.0.tgz", + "integrity": "sha512-zRMhfkByhT2snNdr1si24vJitU6Cr9ix2MikUfWmkAgp4jrNP0GcKSP5YvwQ+TlI8AZXER5QOGJn3JsVtSD9/A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz", + "integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.8.0.tgz", + "integrity": "sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz", + "integrity": "sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.7.4", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.4.tgz", + "integrity": "sha512-tHy0K0VtqNd5Y7Y41h0a0Lhh0L1GzC08dTWg0F7vRJWFtTENg7IZikf3wQkanYIRdb7ngoIPMTmqgUi401fEeQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.35.0", + "@smithy/types": "^4.19.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.19.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.19.0.tgz", + "integrity": "sha512-r7jh49VJxGerfAcTQA6gXcKc+98zOp/tqRwzYjgOE+iSQsP6cEU1hq2QzbuipmP68QtYdY9wKEhiCQZIzHgZ4Q==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -807,6 +1197,12 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.18", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", @@ -2402,8 +2798,7 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD", - "optional": true + "license": "0BSD" }, "node_modules/type-is": { "version": "1.6.18", diff --git a/package.json b/package.json index 8ba6eb4..9d57114 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "start": "node server.js" }, "dependencies": { + "@aws-sdk/client-s3": "^3.1141.0", "bcrypt": "^5.1.1", "express": "^4.21.0", "express-rate-limit": "^8.7.0", diff --git a/public/js/settings.js b/public/js/settings.js index deada3f..2e9e0fe 100644 --- a/public/js/settings.js +++ b/public/js/settings.js @@ -238,6 +238,7 @@ function renderSystemInfo(si) { const photos = si.photos || {}; const files = si.files || {}; const uploads = si.uploads || {}; + const storage = si.storage || {}; const disk = si.disk; let tablesHtml = ''; @@ -283,6 +284,13 @@ function renderSystemInfo(si) {
Файлов${uploads.count || 0}
Размер${esc(uploads.size || '0 B')}
+
+
Хранилище
+
Драйвер${storage.driver === 's3' ? 'S3' : 'Локальное'}
+ ${storage.bucket ? `
Бакет${esc(storage.bucket)}${storage.prefix ? '/' + esc(storage.prefix) : ''}
` : ''} +
Объектов${storage.count || 0}
+
Размер${esc(storage.size || '0 B')}
+
${diskHtml || '
Диск
Недоступно
'} `; } diff --git a/public/version.json b/public/version.json index 57961e3..84941cd 100644 --- a/public/version.json +++ b/public/version.json @@ -1,5 +1,5 @@ { - "full": "e4704c2dc69361318d9aab490128f30c4d875bda", - "short": "e4704c2", - "date": "2026-09-24T11:21:58+03:00" + "full": "8bf54fb95d8d338f35bd6e628e4e21100a5bf371", + "short": "8bf54fb", + "date": "2026-09-24T13:11:31+03:00" } \ No newline at end of file diff --git a/scripts/backup.sh b/scripts/backup.sh index 82b6ed2..ed12de6 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -10,10 +10,13 @@ mkdir -p backups echo ">> Экспорт БД" docker compose exec -T db pg_dump -U app -d whereldo | gzip > backups/db.sql.gz -echo ">> Копирование фотографий" +echo ">> Экспорт файлов из хранилища" rm -rf backups/_uploads mkdir -p backups/_uploads -docker cp "whatido-app-1:/app/uploads/." "backups/_uploads/" +docker compose exec -T app rm -rf /tmp/whatido-export +docker compose exec -T app node scripts/storage-sync.js export /tmp/whatido-export +docker cp "whatido-app-1:/tmp/whatido-export/." "backups/_uploads/" +docker compose exec -T app rm -rf /tmp/whatido-export rm -rf backups/_uploads/.thumbs echo ">> Сборка архива" diff --git a/scripts/migrate-to-s3.js b/scripts/migrate-to-s3.js new file mode 100644 index 0000000..b98d8c5 --- /dev/null +++ b/scripts/migrate-to-s3.js @@ -0,0 +1,128 @@ +const path = require('path'); +const { createStorage } = require('../storage'); + +const SKIP_PREFIXES = ['.thumbs/', '.cache/']; + +function parseArgs(argv) { + const opts = { dryRun: false, deleteLocal: false, verifyOnly: false, concurrency: 3, prefix: null, help: false }; + for (const arg of argv.slice(2)) { + if (arg === '--dry-run') opts.dryRun = true; + else if (arg === '--delete-local') opts.deleteLocal = true; + else if (arg === '--keep-local') opts.deleteLocal = false; + else if (arg === '--verify-only') opts.verifyOnly = true; + else if (arg.startsWith('--concurrency=')) opts.concurrency = Math.max(1, parseInt(arg.split('=')[1], 10) || 1); + else if (arg.startsWith('--prefix=')) opts.prefix = arg.split('=')[1].replace(/^\/+|\/+$/g, ''); + else if (arg === '--help' || arg === '-h') opts.help = true; + } + return opts; +} + +function usage() { + console.log(`Миграция uploads/ -> S3 (MinIO). + +Использование: + node scripts/migrate-to-s3.js [ключи] + +Ключи: + --dry-run показать, что будет загружено, ничего не менять + --verify-only только проверить наличие объектов в S3 + --delete-local удалить локальные файлы после успешной проверки + --keep-local оставить локальные файлы (по умолчанию) + --concurrency=N параллельных загрузок (по умолчанию 3) + --prefix=PREFIX префикс ключей внутри бакета + +Переменные окружения: STORAGE_DRIVER=s3, S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY, S3_SECRET_KEY.`); +} + +async function runPool(items, limit, worker) { + const queue = items.slice(); + const results = { ok: 0, skip: 0, fail: 0 }; + const runners = Array.from({ length: Math.min(limit, queue.length) }, async () => { + while (queue.length) { + const item = queue.shift(); + const res = await worker(item); + if (res === 'ok') results.ok++; + else if (res === 'skip') results.skip++; + else results.fail++; + } + }); + await Promise.all(runners); + return results; +} + +async function main() { + const opts = parseArgs(process.argv); + if (opts.help) return usage(); + const dir = path.join(__dirname, '..', 'uploads'); + const source = createStorage({ dir, driver: 'local' }); + const target = createStorage({ driver: 's3', prefix: opts.prefix || undefined }); + if (!target.isRemote()) { + console.error('STORAGE_DRIVER не s3: задайте STORAGE_DRIVER=s3 и переменные S3_*.'); + process.exit(2); + } + if (!target.bucket) { + console.error('Не задан S3_BUCKET.'); + process.exit(2); + } + + const all = await source.listAll(''); + const objects = all.filter(o => !SKIP_PREFIXES.some(p => o.key.startsWith(p))); + const totalBytes = objects.reduce((sum, o) => sum + (o.size || 0), 0); + if (!opts.dryRun) await target.ensureBucket(); + console.log(`Локально: ${objects.length} файлов, ${(totalBytes / 1048576).toFixed(2)} МБ`); + console.log(`Бакет: ${target.bucket}${opts.prefix ? ' префикс ' + opts.prefix : ''} (${process.env.S3_ENDPOINT || 'endpoint по умолчанию'})`); + if (opts.dryRun) console.log('Режим --dry-run: изменения не применяются.'); + + const failures = []; + const result = await runPool(objects, opts.concurrency, async (obj) => { + const localPath = source.localFile(obj.key); + if (!localPath) { + failures.push(`${obj.key}: некорректный путь`); + return 'fail'; + } + try { + const remote = await target.head(obj.key); + if (remote && remote.size === obj.size) { + if (opts.deleteLocal && !opts.dryRun) source.unlinkLocalOnly(obj.key); + return 'skip'; + } + if (opts.verifyOnly) { + failures.push(`${obj.key}: нет объекта в S3`); + return 'fail'; + } + if (opts.dryRun) { + console.log(` + ${obj.key} (${obj.size} Б)`); + return 'ok'; + } + await target.putFile(obj.key, localPath); + const check = await target.head(obj.key); + if (!check || check.size !== obj.size) { + failures.push(`${obj.key}: размер не совпал после загрузки`); + return 'fail'; + } + if (opts.deleteLocal) source.unlinkLocalOnly(obj.key); + return 'ok'; + } catch (err) { + failures.push(`${obj.key}: ${err.message}`); + return 'fail'; + } + }); + + const usage = await target.usage().catch(() => null); + console.log(`Загружено: ${result.ok}, уже было: ${result.skip}, ошибок: ${result.fail}`); + if (usage) console.log(`В бакете: ${usage.count} объектов, ${(usage.size_bytes / 1048576).toFixed(2)} МБ`); + if (failures.length) { + console.error('Ошибки:'); + for (const f of failures.slice(0, 50)) console.error(' - ' + f); + if (failures.length > 50) console.error(` ... ещё ${failures.length - 50}`); + process.exit(1); + } + if (!opts.deleteLocal && !opts.dryRun) { + console.log('Локальные файлы оставлены. Удалить после проверки: node scripts/migrate-to-s3.js --delete-local'); + } +} + +main().catch(err => { + console.error('Миграция не выполнена:', err.message); + process.exit(1); +}); diff --git a/scripts/restore.sh b/scripts/restore.sh index dbf4983..ef362d8 100755 --- a/scripts/restore.sh +++ b/scripts/restore.sh @@ -28,8 +28,17 @@ echo ">> Восстановление БД (чистая схема)" docker compose exec -T db psql -U app -d whereldo -c 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;' >/dev/null gzip -dc "$TMP/db.sql.gz" | docker compose exec -T db psql -U app -d whereldo -echo ">> Восстановление фотографий" -docker cp "$TMP/_uploads/." "whatido-app-1:/app/uploads/" +echo ">> Загрузка файлов в хранилище" +docker compose start app +for _ in $(seq 1 30); do + docker compose exec -T app true >/dev/null 2>&1 && break + sleep 1 +done +docker compose exec -T app rm -rf /tmp/whatido-import +docker compose exec -T app mkdir -p /tmp/whatido-import +docker cp "$TMP/_uploads/." "whatido-app-1:/tmp/whatido-import/" +docker compose exec -T app node scripts/storage-sync.js import /tmp/whatido-import +docker compose exec -T app rm -rf /tmp/whatido-import echo ">> Запуск приложения" docker compose start app diff --git a/scripts/storage-sync.js b/scripts/storage-sync.js new file mode 100644 index 0000000..7a9e319 --- /dev/null +++ b/scripts/storage-sync.js @@ -0,0 +1,39 @@ +const path = require('path'); +const { createStorage } = require('../storage'); + +function usage() { + console.log(`Экспорт/импорт файлов хранилища (для scripts/backup.sh и scripts/restore.sh). + +Использование: + node scripts/storage-sync.js export <каталог> выгрузить все объекты в каталог + node scripts/storage-sync.js import <каталог> загрузить все файлы из каталога в хранилище + +Работает с текущим драйвером из переменных окружения (local — ./uploads, s3 — бакет MinIO/S3).`); +} + +async function main() { + const [, , mode, dirArg] = process.argv; + if (!mode || !dirArg || mode === '--help' || mode === '-h') { + usage(); + return process.exit(mode ? 0 : 2); + } + const dir = path.resolve(dirArg); + const storage = createStorage({ dir: path.join(__dirname, '..', 'uploads') }); + if (mode === 'export') { + const count = await storage.downloadAll(dir); + console.log(`Экспортировано объектов: ${count} -> ${dir}`); + return; + } + if (mode === 'import') { + const count = await storage.uploadTree(dir); + console.log(`Импортировано файлов: ${count} <- ${dir}`); + return; + } + usage(); + process.exit(2); +} + +main().catch(err => { + console.error('storage-sync failed:', err.message); + process.exit(1); +}); diff --git a/server.js b/server.js index cf7aaeb..a7d9f95 100644 --- a/server.js +++ b/server.js @@ -8,6 +8,8 @@ const heicConvert = require('heic-convert'); const { createEntryAutoChecker, createPhotoEnhanceWorker } = require('./worker'); const { createZipWriter, renderStudentReport } = require('./student-report'); +const { createStorage, mimeFor } = require('./storage'); + const https = require('https'); const path = require('path'); const fs = require('fs'); @@ -17,6 +19,8 @@ types.setTypeParser(1082, v => v); const app = express(); const pool = new Pool({ connectionString: process.env.DATABASE_URL }); +const UPLOADS_DIR = path.join(__dirname, 'uploads'); +const storage = createStorage({ dir: UPLOADS_DIR }); const pgClient = require('pg').Client; const lister = new pgClient({ connectionString: process.env.DATABASE_URL }); @@ -257,8 +261,27 @@ app.use(helmet({ })); app.use(express.json({ limit: '1mb' })); app.use(ipGuard); -const THUMBS_DIR = path.join(__dirname, 'uploads', '.thumbs'); -const ORIGINALS_DIR = path.join(__dirname, 'uploads', '.originals'); +app.use((req, res, next) => { + if (!storage.isRemote()) return next(); + res.on('finish', () => { + if (res.statusCode >= 400) return; + const uploaded = []; + if (req.file) uploaded.push(req.file); + if (Array.isArray(req.files)) uploaded.push(...req.files); + else if (req.files && typeof req.files === 'object') { + for (const list of Object.values(req.files)) if (Array.isArray(list)) uploaded.push(...list); + } + for (const f of uploaded) { + if (!f || !f.filename || typeof f.path !== 'string') continue; + const abs = path.resolve(f.path); + if (!abs.startsWith(UPLOADS_DIR + path.sep)) continue; + storage.persist(f.filename, abs).catch(err => console.error('Upload persist failed:', f.filename, err.message)); + } + }); + next(); +}); +const THUMBS_DIR = path.join(UPLOADS_DIR, '.thumbs'); +const ORIGINALS_DIR = path.join(UPLOADS_DIR, '.originals'); const THUMB_WIDTH = 480; let sharp = null; try { sharp = require('sharp'); } catch {} @@ -266,22 +289,30 @@ if (sharp) { try { fs.mkdirSync(THUMBS_DIR, { recursive: true }); } catch {} } try { fs.mkdirSync(ORIGINALS_DIR, { recursive: true }); } catch {} +try { fs.mkdirSync(storage.cacheDir, { recursive: true }); } catch {} function thumbFileFor(fp) { const base = path.basename(fp).replace(/\.[^.]+$/, '') + '.webp'; return path.join(THUMBS_DIR, base); } -async function sendImageThumb(res, fp) { +function thumbUnlinkFor(key) { + const tp = thumbFileFor(key || ''); + try { if (fs.existsSync(tp)) fs.unlinkSync(tp); } catch {} +} + +async function sendImageThumb(res, key) { + const local = await storage.localize(key); + if (!local) return res.status(404).end(); if (!sharp) { res.setHeader('Cache-Control', 'public, max-age=3600'); - return res.sendFile(fp); + return res.sendFile(local); } - const tp = thumbFileFor(fp); + const tp = thumbFileFor(key); try { if (!fs.existsSync(tp)) { const tmp = tp + '.' + crypto.randomBytes(4).toString('hex') + '.tmp'; - await sharp(fp).rotate().resize({ width: THUMB_WIDTH, withoutEnlargement: true }).webp({ quality: 85 }).toFile(tmp); + await sharp(local).rotate().resize({ width: THUMB_WIDTH, withoutEnlargement: true }).webp({ quality: 85 }).toFile(tmp); fs.renameSync(tmp, tp); } res.setHeader('Cache-Control', 'public, max-age=31536000, immutable'); @@ -289,24 +320,20 @@ async function sendImageThumb(res, fp) { } catch { try { if (fs.existsSync(tp)) fs.unlinkSync(tp); } catch {} res.setHeader('Cache-Control', 'public, max-age=3600'); - return res.sendFile(fp); + return res.sendFile(local); } } app.get('/uploads/thumb/:name', fileLimiter, async (req, res) => { const name = req.params.name; if (!/^[A-Za-z0-9._-]+$/.test(name)) return res.status(400).end(); - const fp = path.join(__dirname, 'uploads', name); - if (!fs.existsSync(fp) || !fs.statSync(fp).isFile()) return res.status(404).end(); - return sendImageThumb(res, fp); + return sendImageThumb(res, name); }); app.get('/uploads/.originals/:name', fileLimiter, async (req, res) => { const name = req.params.name; if (!/^[A-Za-z0-9._-]+$/.test(name)) return res.status(400).end(); - const fp = path.join(ORIGINALS_DIR, name); - if (!fs.existsSync(fp) || !fs.statSync(fp).isFile()) return res.status(404).end(); - return sendImageThumb(res, fp); + return sendImageThumb(res, `.originals/${name}`); }); app.use((req, res, next) => { @@ -317,7 +344,17 @@ app.use((req, res, next) => { next(); }); -app.use('/uploads', express.static(path.join(__dirname, 'uploads'), { maxAge: '365d', immutable: true })); +app.get('/uploads/*', async (req, res) => { + const key = storage.keyFromPath(req.params[0]); + if (!key) return res.status(400).end(); + try { + const ok = await storage.streamTo(res, key, { cacheControl: 'public, max-age=31536000, immutable' }); + if (!ok && !res.headersSent) return res.status(404).end(); + } catch (e) { + console.error('Upload stream failed:', key, e.message); + if (!res.headersSent) res.status(404).end(); + } +}); app.use('/vendor', express.static(path.join(__dirname, 'public', 'vendor'), { maxAge: '30d' })); app.use(express.static(path.join(__dirname, 'public'))); @@ -580,16 +617,12 @@ async function purgeScheduledDeletions() { return { entries: erefs.rowCount, groups: gcount }; } -const UPLOADS_DIR = path.join(__dirname, 'uploads'); - function safeUnlink(relPath) { - if (!relPath || typeof relPath !== 'string') return; - const parts = String(relPath).replace(/\\/g, '/').replace(/^\/+/, '').split('/'); - if (parts[0] === 'uploads') parts.shift(); - if (!parts.length || parts.includes('..') || parts.includes('')) return; - const fp = path.resolve(UPLOADS_DIR, ...parts); - if (fp === UPLOADS_DIR || !fp.startsWith(UPLOADS_DIR + path.sep)) return; - if (fs.existsSync(fp)) fs.unlinkSync(fp); + const key = storage.keyFromPath(relPath); + if (!key) return; + storage.unlinkLocalOnly(key); + if (!storage.isRemote()) return; + storage.del(key).catch(err => console.error('Upload delete failed:', key, err.message)); } function removeUpload(file) { @@ -638,8 +671,8 @@ async function removeEntryFiles(entryId) { } async function sweepOrphanedUploads() { - const dir = path.join(__dirname, 'uploads'); - if (!fs.existsSync(dir)) return; + const dir = UPLOADS_DIR; + try { fs.mkdirSync(dir, { recursive: true }); } catch {} const [{ rows: photos }, { rows: files }, { rows: gphotos }, { rows: ephotos }, { rows: pendingJobs }, { rows: mphotos }, { rows: sphotos }, { rows: logos }] = await Promise.all([ pool.query('SELECT photo_path AS p FROM entries WHERE photo_path IS NOT NULL'), pool.query('SELECT path AS p FROM project_files'), @@ -651,14 +684,28 @@ async function sweepOrphanedUploads() { pool.query(`SELECT value AS p FROM settings WHERE key = 'system_logo' AND value IS NOT NULL AND value <> ''`), ]); const refs = new Set(); - [...photos, ...files, ...gphotos, ...ephotos, ...pendingJobs, ...mphotos, ...sphotos, ...logos].forEach(r => refs.add('/' + String(r.p).replace(/^\/+/, ''))); + [...photos, ...files, ...gphotos, ...ephotos, ...pendingJobs, ...mphotos, ...sphotos, ...logos].forEach(r => { + const key = storage.keyFromPath(r.p); + if (key) refs.add(key); + }); + const remoteObjects = await storage.listAll('').catch(err => { + console.error('Storage list failed:', err.message); + return []; + }); + for (const obj of remoteObjects) { + if (!storage.normalizeKey(obj.key) || obj.key.includes('/')) continue; + if (!refs.has(obj.key)) { + try { await storage.del(obj.key); } catch (err) { console.error('Upload delete failed:', obj.key, err.message); } + } + } for (const f of fs.readdirSync(dir)) { const fp = path.join(dir, f); if (!fs.statSync(fp).isFile()) continue; - if (!refs.has('/uploads/' + f)) { + if (!refs.has(f)) { try { fs.unlinkSync(fp); } catch {} } } + storage.pruneCache(); } async function ensureAuditTable() { @@ -1704,22 +1751,7 @@ async function buildBackupArchive() { st.rows.forEach(r => { settings[r.key] = r.value; }); const payload = { version: 1, created_at: new Date().toISOString(), groups: g.rows, students: s.rows, entries: e.rows, settings, project_files: pf.rows, branches: br.rows, users: us.rows, user_branches: ub.rows, group_photos: gp.rows, entry_photos: ep.rows, modules: md.rows, student_photos: sp.rows, share_links: sl.rows, photo_jobs: pj.rows }; fs.writeFileSync(path.join(staging, 'data.json'), JSON.stringify(payload)); - fs.mkdirSync(path.join(staging, 'uploads'), { recursive: true }); - const dir = path.join(__dirname, 'uploads'); - if (fs.existsSync(dir)) { - for (const f of fs.readdirSync(dir)) { - const fp = path.join(dir, f); - if (fs.statSync(fp).isFile() && SAFE_NAME.test(f)) fs.copyFileSync(fp, path.join(staging, 'uploads', f)); - } - const orig = path.join(dir, '.originals'); - if (fs.existsSync(orig)) { - fs.mkdirSync(path.join(staging, 'uploads', '.originals'), { recursive: true }); - for (const f of fs.readdirSync(orig)) { - const ofp = path.join(orig, f); - if (fs.statSync(ofp).isFile() && SAFE_NAME.test(f)) fs.copyFileSync(ofp, path.join(staging, 'uploads', '.originals', f)); - } - } - } + await storage.downloadAll(path.join(staging, 'uploads')); const stamp = new Date().toISOString().slice(0, 16).replace(/[:T]/g, '-'); fs.mkdirSync(BACKUP_DIR, { recursive: true }); const outPath = path.join(BACKUP_DIR, `whatido-backup-${stamp}-${crypto.randomBytes(4).toString('hex')}.tar.gz`); @@ -1971,31 +2003,14 @@ app.post('/api/restore', requireAdmin, uploadBackup.single('backup'), async (req } finally { client.release(); } - const dir = path.join(__dirname, 'uploads'); - fs.mkdirSync(dir, { recursive: true }); + fs.mkdirSync(UPLOADS_DIR, { recursive: true }); if (legacyPhotos.length) { for (const p of legacyPhotos) { if (!p.path || !SAFE_NAME.test(p.path)) continue; - fs.writeFileSync(path.join(dir, p.path), Buffer.from(p.data, 'base64')); + await storage.put(p.path, Buffer.from(p.data, 'base64')); } } else { - const src = path.join(staging, 'uploads'); - if (fs.existsSync(src)) { - for (const f of fs.readdirSync(src)) { - if (!SAFE_NAME.test(f)) continue; - const fp = path.join(src, f); - if (fs.statSync(fp).isFile()) fs.copyFileSync(fp, path.join(dir, f)); - } - const orgSrc = path.join(src, '.originals'); - if (fs.existsSync(orgSrc)) { - fs.mkdirSync(path.join(dir, '.originals'), { recursive: true }); - for (const f of fs.readdirSync(orgSrc)) { - if (!SAFE_NAME.test(f)) continue; - const ofp = path.join(orgSrc, f); - if (fs.statSync(ofp).isFile()) fs.copyFileSync(ofp, path.join(dir, '.originals', f)); - } - } - } + await storage.uploadTree(path.join(staging, 'uploads')); } fs.rmSync(staging, { recursive: true, force: true }); cleanupUpload(req); @@ -2315,14 +2330,17 @@ app.get('/api/share/:shareToken/files/:fileToken', fileLimiter, async (req, res) ); if (!rows.length) return res.status(404).json({ error: 'Not found' }); const r = rows[0]; - const fp = path.join(__dirname, r.path); - if (!fs.existsSync(fp)) return res.status(404).json({ error: 'File missing' }); + const key = storage.keyFromPath(r.path); + if (!key) return res.status(404).json({ error: 'File missing' }); if (isImageName(r.name)) { - if (req.query.thumb) return sendImageThumb(res, fp); - res.setHeader('Cache-Control', 'public, max-age=31536000, immutable'); - return res.sendFile(fp); + if (req.query.thumb) return sendImageThumb(res, key); + const ok = await storage.streamTo(res, key, { cacheControl: 'public, max-age=31536000, immutable' }); + if (!ok && !res.headersSent) return res.status(404).json({ error: 'File missing' }); + return; + } + if (!(await storage.streamTo(res, key, { download: true, name: r.name })) && !res.headersSent) { + return res.status(404).json({ error: 'File missing' }); } - return res.download(fp, r.name); }); app.get('/s/:token', (req, res) => { @@ -3332,15 +3350,14 @@ app.get('/api/export/student', requireAuth, async (req, res) => { const seenPhotos = new Set(); const photosBuilt = []; - function addPhoto(p) { + async function addPhoto(p) { if (!isSafeUploadPath(p.photo_path)) return; const stored = p.photo_path.slice('/uploads/'.length); if (seenPhotos.has(stored)) return; seenPhotos.add(stored); - const src = path.join(UPLOADS_DIR, stored); - if (!fs.existsSync(src)) return; + const data = await storage.getBuffer(stored); + if (!data) return; const ts = p.created_at ? new Date(p.created_at) : new Date(); - const data = fs.readFileSync(src); zip.addFile('photos/' + stored, data, ts); photosBuilt.push({ stored, @@ -3356,13 +3373,13 @@ app.get('/api/export/student', requireAuth, async (req, res) => { const profilePhoto = (student.profile && student.profile.photo_path) || student.photo_path; if (profilePhoto && isSafeUploadPath(profilePhoto)) { const stored = profilePhoto.slice('/uploads/'.length); - if (fs.existsSync(path.join(UPLOADS_DIR, stored))) avatarStored = stored; + if (await storage.exists(stored)) avatarStored = stored; } - for (const p of photosRes.rows) addPhoto(p); - for (const m of mainsRes.rows) addPhoto(m); + for (const p of photosRes.rows) await addPhoto(p); + for (const m of mainsRes.rows) await addPhoto(m); photosBuilt.sort((a, b) => new Date(b.createdAt || Date.now()) - new Date(a.createdAt || Date.now())); if (avatarStored) { - if (!seenPhotos.has(avatarStored)) addPhoto({ photo_path: '/uploads/' + avatarStored, caption: 'Фото резидента', created_at: student.created_at }); + if (!seenPhotos.has(avatarStored)) await addPhoto({ photo_path: '/uploads/' + avatarStored, caption: 'Фото резидента', created_at: student.created_at }); const idx = photosBuilt.findIndex(p => p.stored === avatarStored); if (idx > 0) photosBuilt.unshift(photosBuilt.splice(idx, 1)[0]); } @@ -3374,10 +3391,10 @@ app.get('/api/export/student', requireAuth, async (req, res) => { if (!isSafeUploadPath(gp.photo_path)) continue; const stored = gp.photo_path.slice('/uploads/'.length); if (seenPhotos.has(stored)) continue; - const src = path.join(UPLOADS_DIR, stored); - if (!fs.existsSync(src)) continue; + const data = await storage.getBuffer(stored); + if (!data) continue; const ts = gp.taken_at || gp.created_at || new Date(); - zip.addFile('photos/' + stored, fs.readFileSync(src), new Date(ts)); + zip.addFile('photos/' + stored, data, new Date(ts)); seenPhotos.add(stored); groupPhotosBuilt.push({ stored, @@ -3395,9 +3412,8 @@ app.get('/api/export/student', requireAuth, async (req, res) => { for (const f of filesRes.rows) { if (!isSafeUploadPath(f.path)) continue; const stored = f.path.slice('/uploads/'.length); - const src = path.join(UPLOADS_DIR, stored); - if (!fs.existsSync(src)) continue; - const data = fs.readFileSync(src); + const data = await storage.getBuffer(stored); + if (!data) continue; let base = String(f.name || 'file').replace(/[\\/:*?"<>|]/g, '_').replace(/^[.\s]+/, '').slice(0, 120) || 'file'; const ext = path.extname(base); const stem = ext ? base.slice(0, -ext.length) : base; @@ -3700,11 +3716,10 @@ app.get('/api/files', requireAuth, async (req, res) => { const off = parseInt(offset, 10); if (off > 0) { qparams.push(off); q += ` OFFSET $${qparams.length}`; } const { rows } = await pool.query(q, qparams); - const files = rows.map(r => { - let size = 0; - try { size = fs.statSync(path.join(__dirname, r.path)).size; } catch {} + const files = await Promise.all(rows.map(async r => { + const size = await storage.sizeOf(r.path); return { id: r.id, token: r.token, name: r.name, student_name: r.student_name, group_name: r.group_name, created_at: r.created_at, size }; - }); + })); res.json({ files, total }); }); @@ -3728,11 +3743,10 @@ app.get('/api/files/detached', requireAdmin, async (req, res) => { const off = parseInt(offset, 10); if (off > 0) { qparams.push(off); q += ` OFFSET $${qparams.length}`; } const { rows } = await pool.query(q, qparams); - const files = rows.map(r => { - let size = 0; - try { size = fs.statSync(path.join(__dirname, r.path)).size; } catch {} + const files = await Promise.all(rows.map(async r => { + const size = await storage.sizeOf(r.path); return { id: r.id, token: r.token, name: r.name, created_at: r.created_at, size }; - }); + })); res.json({ files, total }); }); @@ -3783,14 +3797,17 @@ app.get('/api/files/:token', fileLimiter, async (req, res) => { const { rows } = await pool.query('SELECT path, name FROM project_files WHERE token = $1', [req.params.token]); if (!rows.length) return res.status(404).json({ error: 'Not found' }); const r = rows[0]; - const fp = path.join(__dirname, r.path); - if (!fs.existsSync(fp)) return res.status(404).json({ error: 'File missing' }); + const key = storage.keyFromPath(r.path); + if (!key) return res.status(404).json({ error: 'File missing' }); if (isImageName(r.name)) { - if (req.query.thumb) return sendImageThumb(res, fp); - res.setHeader('Cache-Control', 'public, max-age=31536000, immutable'); - return res.sendFile(fp); + if (req.query.thumb) return sendImageThumb(res, key); + const ok = await storage.streamTo(res, key, { cacheControl: 'public, max-age=31536000, immutable' }); + if (!ok && !res.headersSent) return res.status(404).json({ error: 'File missing' }); + return; + } + if (!(await storage.streamTo(res, key, { download: true, name: r.name })) && !res.headersSent) { + return res.status(404).json({ error: 'File missing' }); } - return res.download(fp, r.name); }); app.get('/api/stats', requireAuth, async (req, res) => { @@ -3871,36 +3888,23 @@ app.get('/api/system-info', requireAdmin, async (_, res) => { WHERE photo_path IS NOT NULL AND deleted_at IS NULL `); - function sumPhotoSizes(paths) { + async function sumPhotoSizes(paths) { let bytes = 0; for (const p of paths) { if (!p) continue; - const fp = path.join(__dirname, p); - try { - const st = fs.statSync(fp); - if (st.isFile()) bytes += st.size; - } catch {} + bytes += await storage.sizeOf(p); } return bytes; } const groupPhotoSizes = await pool.query('SELECT photo_path FROM group_photos'); const entryPhotoSizes = await pool.query('SELECT photo_path FROM entries WHERE photo_path IS NOT NULL AND deleted_at IS NULL'); - const groupPhotoBytes = sumPhotoSizes(groupPhotoSizes.rows.map(r => r.photo_path)); - const entryPhotoBytes = sumPhotoSizes(entryPhotoSizes.rows.map(r => r.photo_path)); + const groupPhotoBytes = await sumPhotoSizes(groupPhotoSizes.rows.map(r => r.photo_path)); + const entryPhotoBytes = await sumPhotoSizes(entryPhotoSizes.rows.map(r => r.photo_path)); - const uploadsDir = path.join(__dirname, 'uploads'); - let uploadsSize = 0; - let uploadsCount = 0; - if (fs.existsSync(uploadsDir)) { - for (const f of fs.readdirSync(uploadsDir)) { - const fp = path.join(uploadsDir, f); - if (fs.statSync(fp).isFile()) { - uploadsCount++; - uploadsSize += fs.statSync(fp).size; - } - } - } + const usage = await storage.usage(); + const uploadsSize = usage.size_bytes; + const uploadsCount = usage.count; const diskInfo = getDiskInfo(); @@ -3929,6 +3933,14 @@ app.get('/api/system-info', requireAdmin, async (_, res) => { size: formatBytes(uploadsSize), size_bytes: uploadsSize, }, + storage: { + driver: storage.isRemote() ? 's3' : 'local', + bucket: usage.bucket, + prefix: usage.prefix, + count: usage.count, + size: formatBytes(usage.size_bytes), + size_bytes: usage.size_bytes, + }, disk: diskInfo, }; }); @@ -4285,26 +4297,26 @@ async function swapEntryPhotoFiles(req, entryId, oldPath, newPath, { keepOrigina let originalPath = null; const { rows: prevRows } = await pool.query('SELECT photo_original_path FROM entries WHERE id = $1', [entryId]); const prevOriginal = prevRows.length ? prevRows[0].photo_original_path : null; - const oldAbs = path.join(UPLOADS_DIR, String(oldPath).replace(/^\/+/, '').replace(/^uploads\//, '')); + const oldKey = storage.keyFromPath(oldPath); if (prevOriginal) { originalPath = prevOriginal; - try { if (fs.existsSync(oldAbs)) fs.unlinkSync(oldAbs); } catch {} - } else if (keepOriginal && fs.existsSync(oldAbs)) { + if (oldKey) safeUnlink(oldKey); + } else if (keepOriginal && oldKey && (await storage.exists(oldKey))) { try { - const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg'); - const backupPath = path.join(ORIGINALS_DIR, backupName); - fs.renameSync(oldAbs, backupPath); - originalPath = `/uploads/.originals/${backupName}`; + const backupKey = `.originals/${crypto.randomBytes(12).toString('hex')}${path.extname(oldKey) || '.jpg'}`; + if (await storage.copyObject(oldKey, backupKey)) { + originalPath = `/uploads/${backupKey}`; + safeUnlink(oldKey); + } } catch (e) { console.error('photo original backup failed:', e); } - } else { - try { if (fs.existsSync(oldAbs)) fs.unlinkSync(oldAbs); } catch {} + } else if (oldKey) { + safeUnlink(oldKey); } await pool.query('UPDATE entries SET photo_path = $1, photo_original_path = $2 WHERE id = $3', [newPath, originalPath, entryId]); await pool.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [newPath, entryId, oldPath]); - const oldThumb = path.join('uploads', '.thumbs', path.basename(oldPath).replace(/\.[^.]+$/, '') + '.webp'); - safeUnlink(oldThumb); + thumbUnlinkFor(oldPath); await pool.query( 'INSERT INTO photo_jobs (entry_id, action, status, before_path, after_path, finished_at) VALUES ($1, $2, $3, $4, $5, now())', [entryId, action, 'done', originalPath, newPath] @@ -4343,7 +4355,9 @@ app.put('/api/entries/:id/photo/enhance', requireAuth, (req, res, next) => { const saturate = clampEnhanceParam(p.saturate, 0, 300, 100); const sharpAmt = clampEnhanceParam(p.sharp, 0, 100, 0); const denoise = clampEnhanceParam(p.denoise, 0, 100, 0); - const srcPath = path.join(__dirname, oldPath.replace(/^\/+/, '')); + const srcKey = storage.keyFromPath(oldPath); + const srcPath = srcKey ? await storage.localize(srcKey) : null; + if (!srcPath) return res.status(400).json({ error: 'Файл фото не найден' }); let pipeline = sharp(srcPath).rotate(); if (denoise > 0) pipeline = pipeline.median(denoise > 70 ? 5 : 3); pipeline = pipeline.modulate({ brightness: brightness / 100, saturation: saturate / 100 }); @@ -4355,8 +4369,9 @@ app.put('/api/entries/:id/photo/enhance', requireAuth, (req, res, next) => { pipeline = pipeline.sharpen({ sigma: 0.5 + (sharpAmt / 100) * 1.5, m1: 0, m2: 1 + sharpAmt / 50 }); } const newName = crypto.randomBytes(12).toString('hex') + '.jpg'; - const outPath = path.join(__dirname, 'uploads', newName); + const outPath = path.join(UPLOADS_DIR, newName); await pipeline.jpeg({ quality: 92, mozjpeg: true }).toFile(outPath); + await storage.persist(newName, outPath); newPath = `/uploads/${newName}`; engine = 'sharp'; } else { @@ -4436,8 +4451,7 @@ app.post('/api/entries/:id/photo/jobs/:jobId/apply', requireAuth, async (req, re await client.query('ROLLBACK'); return res.status(400).json({ error: 'Некорректный путь результата' }); } - const afterAbs = path.join(UPLOADS_DIR, String(job.after_path).replace(/^\/+/, '').replace(/^uploads\//, '')); - if (!fs.existsSync(afterAbs)) { + if (!(await storage.exists(job.after_path))) { await client.query('ROLLBACK'); return res.status(400).json({ error: 'Файл результата не найден' }); } @@ -4451,21 +4465,22 @@ app.post('/api/entries/:id/photo/jobs/:jobId/apply', requireAuth, async (req, re } let originalPath = entryRows[0].photo_original_path; let beforePath = originalPath; - const oldAbs = path.join(UPLOADS_DIR, String(oldPath || '').replace(/^\/+/, '').replace(/^uploads\//, '')); - if (oldPath && fs.existsSync(oldAbs)) { + const oldKey = oldPath ? storage.keyFromPath(oldPath) : null; + if (oldKey && (await storage.exists(oldKey))) { try { - const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg'); - fs.renameSync(oldAbs, path.join(ORIGINALS_DIR, backupName)); - beforePath = `/uploads/.originals/${backupName}`; - if (!originalPath) originalPath = beforePath; + const backupKey = `.originals/${crypto.randomBytes(12).toString('hex')}${path.extname(oldKey) || '.jpg'}`; + if (await storage.copyObject(oldKey, backupKey)) { + beforePath = `/uploads/${backupKey}`; + if (!originalPath) originalPath = beforePath; + safeUnlink(oldKey); + } } catch (e) { console.error('photo apply backup failed:', e); } } await client.query('UPDATE entries SET photo_path = $1, photo_original_path = $2 WHERE id = $3', [job.after_path, originalPath, req.params.id]); await client.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [job.after_path, req.params.id, oldPath]); - const oldThumb = path.join('uploads', '.thumbs', path.basename(oldPath || '').replace(/\.[^.]+$/, '') + '.webp'); - safeUnlink(oldThumb); + thumbUnlinkFor(oldPath); await client.query(`UPDATE photo_jobs SET applied = true, before_path = COALESCE(before_path, $1) WHERE id = $2`, [beforePath, jobId]); await client.query('COMMIT'); await logAudit(req, 'entry.photo.apply', { entry_id: req.params.id, job_id: jobId, before_path: beforePath, after_path: job.after_path }); @@ -4544,28 +4559,32 @@ app.post('/api/entries/:id/photo/jobs/:jobId/rollback', requireAuth, async (req, } const beforeName = path.basename(job.before_path); if (!/^[A-Za-z0-9._-]+$/.test(beforeName)) return res.status(400).json({ error: 'Некорректный путь' }); - const beforeAbs = path.join(ORIGINALS_DIR, beforeName); - if (!fs.existsSync(beforeAbs)) return res.status(400).json({ error: 'Файл версии не найден' }); + const beforeKey = `.originals/${beforeName}`; + if (!(await storage.exists(beforeKey))) return res.status(400).json({ error: 'Файл версии не найден' }); const { rows: cur } = await pool.query('SELECT photo_path FROM entries WHERE id = $1', [req.params.id]); if (!cur.length) return res.status(404).json({ error: 'Запись не найдена' }); const oldPath = cur[0].photo_path; if (!oldPath) return res.status(400).json({ error: 'У записи нет фото' }); const ext = path.extname(beforeName) || '.jpg'; - const newPath = `/uploads/${crypto.randomBytes(12).toString('hex')}${ext}`; - fs.copyFileSync(beforeAbs, path.join(UPLOADS_DIR, path.basename(newPath))); - const oldAbs = path.join(UPLOADS_DIR, String(oldPath).replace(/^\/+/, '').replace(/^uploads\//, '')); + const newKey = `${crypto.randomBytes(12).toString('hex')}${ext}`; + const newPath = `/uploads/${newKey}`; + if (!(await storage.copyObject(beforeKey, newKey))) { + return res.status(400).json({ error: 'Файл версии не найден' }); + } + const oldKey = storage.keyFromPath(oldPath); let rollbackBefore = null; - if (fs.existsSync(oldAbs)) { + if (oldKey && (await storage.exists(oldKey))) { try { - const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg'); - fs.renameSync(oldAbs, path.join(ORIGINALS_DIR, backupName)); - rollbackBefore = `/uploads/.originals/${backupName}`; + const backupKey = `.originals/${crypto.randomBytes(12).toString('hex')}${path.extname(oldKey) || '.jpg'}`; + if (await storage.copyObject(oldKey, backupKey)) { + rollbackBefore = `/uploads/${backupKey}`; + safeUnlink(oldKey); + } } catch (e) { console.error('photo rollback backup failed:', e); } } - const oldThumb = path.join('uploads', '.thumbs', path.basename(oldPath).replace(/\.[^.]+$/, '') + '.webp'); - safeUnlink(oldThumb); + thumbUnlinkFor(oldPath); await pool.query('UPDATE entries SET photo_path = $1 WHERE id = $2', [newPath, req.params.id]); await pool.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [newPath, req.params.id, oldPath]); await pool.query( @@ -4606,26 +4625,29 @@ app.post('/api/entries/:id/photo/restore-original', requireAuth, async (req, res if (!origPath) return res.status(400).json({ error: 'Оригинал не сохранён' }); const origName = path.basename(origPath); if (!/^[A-Za-z0-9._-]+$/.test(origName)) return res.status(400).json({ error: 'Некорректный путь оригинала' }); - const origAbs = path.join(ORIGINALS_DIR, origName); - if (!fs.existsSync(origAbs)) return res.status(400).json({ error: 'Файл оригинала не найден' }); - const newPath = `/uploads/${crypto.randomBytes(12).toString('hex')}${path.extname(origName) || '.jpg'}`; - fs.renameSync(origAbs, path.join(UPLOADS_DIR, path.basename(newPath))); + const origKey = `.originals/${origName}`; + if (!(await storage.exists(origKey))) return res.status(400).json({ error: 'Файл оригинала не найден' }); + const newKey = `${crypto.randomBytes(12).toString('hex')}${path.extname(origName) || '.jpg'}`; + const newPath = `/uploads/${newKey}`; + if (!(await storage.copyObject(origKey, newKey))) return res.status(400).json({ error: 'Файл оригинала не найден' }); + await storage.del(origKey); const oldPath = rows[0].photo_path; await pool.query('UPDATE entries SET photo_path = $1, photo_original_path = NULL WHERE id = $2', [newPath, req.params.id]); await pool.query('UPDATE entry_photos SET photo_path = $1 WHERE entry_id = $2 AND photo_path = $3', [newPath, req.params.id, oldPath]); - const oldAbs = path.join(UPLOADS_DIR, String(oldPath).replace(/^\/+/, '').replace(/^uploads\//, '')); + const oldKey = storage.keyFromPath(oldPath); let beforePath = null; - if (fs.existsSync(oldAbs)) { + if (oldKey && (await storage.exists(oldKey))) { try { - const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg'); - fs.renameSync(oldAbs, path.join(ORIGINALS_DIR, backupName)); - beforePath = `/uploads/.originals/${backupName}`; + const backupKey = `.originals/${crypto.randomBytes(12).toString('hex')}${path.extname(oldKey) || '.jpg'}`; + if (await storage.copyObject(oldKey, backupKey)) { + beforePath = `/uploads/${backupKey}`; + safeUnlink(oldKey); + } } catch (e) { console.error('photo original backup failed:', e); } } - const oldThumb = path.join('uploads', '.thumbs', path.basename(oldPath).replace(/\.[^.]+$/, '') + '.webp'); - safeUnlink(oldThumb); + thumbUnlinkFor(oldPath); await pool.query( 'INSERT INTO photo_jobs (entry_id, action, status, before_path, after_path, error, finished_at) VALUES ($1, $2, $3, $4, $5, NULL, now())', [req.params.id, 'restore', 'done', beforePath, newPath] @@ -5250,8 +5272,19 @@ if (fs.existsSync(certPath) && fs.existsSync(keyPath)) { try { await pool.query(`INSERT INTO settings (key, value) VALUES ('camera_enabled', 'true') ON CONFLICT (key) DO NOTHING`); } catch (err) { console.error('Camera setting:', err); } try { await pool.query(`INSERT INTO settings (key, value) VALUES ('trash_purge_days', '30') ON CONFLICT (key) DO NOTHING`); } catch (err) { console.error('Trash purge days setting:', err); } try { await sweepOrphanedUploads(); } catch (err) { console.error('Upload sweep:', err); } + if (storage.isRemote()) { + try { + await storage.ensureBucket(); + console.log('S3 bucket ready:', storage.bucket); + } catch (err) { + console.error('S3 bucket init:', err.message); + } + } try { await purgeScheduledDeletions(); } catch (err) { console.error('Trash purge:', err); } setInterval(() => { purgeScheduledDeletions().catch(err => console.error('Trash purge:', err)); }, 60 * 60 * 1000).unref(); + setInterval(() => { + try { storage.pruneCache(); } catch (err) { console.error('Cache prune:', err); } + }, 60 * 60 * 1000).unref(); entryAutoChecker = createEntryAutoChecker({ pool, getSetting, logAudit, aiUrl: AI_URL, defaultPrompt: AI_DEFAULT_PROMPT }); entryAutoChecker.start(); console.log('AI auto-check worker started'); @@ -5264,6 +5297,7 @@ if (fs.existsSync(certPath) && fs.existsSync(keyPath)) { photoAiUrl: PHOTO_AI_URL, uploadsDir: UPLOADS_DIR, originalsDir: ORIGINALS_DIR, + storage, }); photoWorker.start(); console.log('Photo enhance worker started'); diff --git a/storage.js b/storage.js new file mode 100644 index 0000000..f6b6609 --- /dev/null +++ b/storage.js @@ -0,0 +1,551 @@ +const fs = require('fs'); +const path = require('path'); +const { pipeline } = require('stream/promises'); +const { + S3Client, + PutObjectCommand, + GetObjectCommand, + HeadObjectCommand, + DeleteObjectCommand, + CopyObjectCommand, + ListObjectsV2Command, + HeadBucketCommand, + CreateBucketCommand, +} = require('@aws-sdk/client-s3'); + +const MIME_TYPES = { + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.jfif': 'image/jpeg', + '.png': 'image/png', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.bmp': 'image/bmp', + '.avif': 'image/avif', + '.ico': 'image/x-icon', + '.heic': 'image/heic', + '.heif': 'image/heif', + '.pdf': 'application/pdf', + '.txt': 'text/plain; charset=utf-8', + '.md': 'text/markdown; charset=utf-8', + '.html': 'text/html; charset=utf-8', + '.htm': 'text/html; charset=utf-8', + '.zip': 'application/zip', + '.rar': 'application/vnd.rar', + '.7z': 'application/x-7z-compressed', + '.doc': 'application/msword', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', +}; + +const SAFE_SEGMENT = /^[\w,.()-]+$/; +const NON_OBJECT_PREFIXES = ['.thumbs/', '.cache/']; + +function mimeFor(name) { + const m = String(name || '').toLowerCase().match(/\.[a-z0-9]+$/); + return (m && MIME_TYPES[m[0]]) || 'application/octet-stream'; +} + +function normalizeKey(input) { + if (typeof input !== 'string' || !input) return null; + let key = input.replace(/\\/g, '/').replace(/^\/+/, ''); + if (key.startsWith('uploads/')) key = key.slice('uploads/'.length); + if (!key || key.length > 255 || key.includes('..')) return null; + const segments = key.split('/'); + if (segments.some(seg => !SAFE_SEGMENT.test(seg))) return null; + return key; +} + +function isObjectKey(key) { + return !NON_OBJECT_PREFIXES.some(p => key.startsWith(p)); +} + +function createStorage(options = {}) { + const driver = options.driver || process.env.STORAGE_DRIVER || 'local'; + const dir = path.resolve(options.dir || path.join(__dirname, 'uploads')); + const cacheDir = path.resolve(options.cacheDir || path.join(dir, '.cache')); + const localFallback = options.localFallback !== undefined + ? !!options.localFallback + : process.env.STORAGE_LOCAL_FALLBACK !== '0'; + const keepLocal = options.keepLocal !== undefined + ? !!options.keepLocal + : process.env.STORAGE_KEEP_LOCAL === '1'; + const bucket = options.bucket || process.env.S3_BUCKET || ''; + const prefix = String(options.prefix !== undefined ? options.prefix : process.env.S3_PREFIX || '').replace(/^\/+|\/+$/g, ''); + const cacheMaxAgeMs = Math.max(0, parseInt(options.cacheMaxAgeHours || process.env.STORAGE_CACHE_MAX_AGE_HOURS || '168', 10) || 0) * 3600 * 1000; + const remote = driver === 's3'; + const client = remote + ? new S3Client({ + region: options.region || process.env.S3_REGION || 'us-east-1', + endpoint: options.endpoint || process.env.S3_ENDPOINT || undefined, + forcePathStyle: options.forcePathStyle !== undefined + ? !!options.forcePathStyle + : process.env.S3_FORCE_PATH_STYLE !== '0', + credentials: { + accessKeyId: options.accessKey || process.env.S3_ACCESS_KEY || '', + secretAccessKey: options.secretKey || process.env.S3_SECRET_KEY || '', + }, + }) + : null; + + function isRemote() { + return remote; + } + + function objectKey(key) { + const k = normalizeKey(key); + if (!k) return null; + return prefix ? `${prefix}/${k}` : k; + } + + function localFile(key) { + const k = normalizeKey(key); + if (!k) return null; + const abs = path.resolve(dir, ...k.split('/')); + if (abs !== dir && !abs.startsWith(dir + path.sep)) return null; + return abs; + } + + function cacheFile(key) { + const k = normalizeKey(key); + if (!k) return null; + return path.join(cacheDir, ...k.split('/')); + } + + function publicPath(key) { + const k = normalizeKey(key); + return k ? `/uploads/${k}` : null; + } + + function keyFromPath(p) { + return normalizeKey(p); + } + + function localExists(key) { + const fp = localFile(key); + if (!fp) return false; + try { + return fs.statSync(fp).isFile(); + } catch { + return false; + } + } + + function unlinkLocalOnly(key) { + for (const fp of [localFile(key), cacheFile(key)]) { + if (!fp) continue; + try { + if (fs.existsSync(fp)) fs.unlinkSync(fp); + } catch {} + } + } + + function isMissingError(err) { + if (!err) return false; + const code = err.name || err.Code || err.code; + const status = err.$metadata && err.$metadata.httpStatusCode; + return code === 'NoSuchKey' || code === 'NotFound' || code === 'ENOENT' || status === 404; + } + + async function put(key, body, opts = {}) { + const objKey = objectKey(key); + if (!objKey) return false; + if (!remote) { + const fp = localFile(key); + if (!fp) return false; + fs.mkdirSync(path.dirname(fp), { recursive: true }); + if (Buffer.isBuffer(body) || typeof body === 'string') fs.writeFileSync(fp, body); + else await pipeline(body, fs.createWriteStream(fp)); + return true; + } + await client.send(new PutObjectCommand({ + Bucket: bucket, + Key: objKey, + Body: body, + ContentType: opts.contentType || mimeFor(key), + ContentLength: opts.contentLength, + })); + return true; + } + + async function putFile(key, localPath, opts = {}) { + const objKey = objectKey(key); + if (!objKey) return false; + let size = null; + try { + size = fs.statSync(localPath).size; + } catch { + return false; + } + if (!remote) { + const fp = localFile(key); + if (!fp) return false; + fs.mkdirSync(path.dirname(fp), { recursive: true }); + if (path.resolve(localPath) !== fp) fs.copyFileSync(localPath, fp); + return true; + } + await client.send(new PutObjectCommand({ + Bucket: bucket, + Key: objKey, + Body: fs.createReadStream(localPath), + ContentLength: size, + ContentType: opts.contentType || mimeFor(key), + })); + return true; + } + + async function head(key) { + const objKey = objectKey(key); + if (!objKey) return null; + if (!remote) { + const fp = localFile(key); + if (!fp) return null; + try { + const st = fs.statSync(fp); + if (!st.isFile()) return null; + return { key: normalizeKey(key), size: st.size, contentType: mimeFor(key), etag: null }; + } catch { + return null; + } + } + try { + const out = await client.send(new HeadObjectCommand({ Bucket: bucket, Key: objKey })); + return { + key: normalizeKey(key), + size: out.ContentLength || 0, + contentType: out.ContentType || mimeFor(key), + etag: out.ETag ? out.ETag.replace(/"/g, '') : null, + }; + } catch (err) { + if (isMissingError(err)) return null; + throw err; + } + } + + async function exists(key) { + if (localFallback && localExists(key)) return true; + return !!(await head(key)); + } + + async function sizeOf(key) { + const info = await head(key); + return info ? info.size : 0; + } + + async function getStream(key) { + const objKey = objectKey(key); + if (!objKey) return null; + if (!remote) { + const fp = localFile(key); + if (!fp || !fs.existsSync(fp)) return null; + return { stream: fs.createReadStream(fp), contentLength: fs.statSync(fp).size, contentType: mimeFor(key) }; + } + try { + const out = await client.send(new GetObjectCommand({ Bucket: bucket, Key: objKey })); + return { stream: out.Body, contentLength: out.ContentLength || 0, contentType: out.ContentType || mimeFor(key) }; + } catch (err) { + if (isMissingError(err)) return null; + throw err; + } + } + + async function getBuffer(key) { + const fp = await localize(key); + if (fp) return fs.readFileSync(fp); + return null; + } + + async function del(key) { + const k = normalizeKey(key); + if (!k) return false; + unlinkLocalOnly(k); + if (!remote) return true; + const objKey = objectKey(k); + try { + await client.send(new DeleteObjectCommand({ Bucket: bucket, Key: objKey })); + } catch (err) { + if (!isMissingError(err)) throw err; + } + return true; + } + + async function copyObject(srcKey, dstKey) { + const src = normalizeKey(srcKey); + const dst = normalizeKey(dstKey); + if (!src || !dst) return false; + if (!remote) { + const from = localFile(src); + const to = localFile(dst); + if (!from || !to || !fs.existsSync(from)) return false; + fs.mkdirSync(path.dirname(to), { recursive: true }); + fs.copyFileSync(from, to); + return true; + } + try { + await client.send(new CopyObjectCommand({ + Bucket: bucket, + Key: objectKey(dst), + CopySource: `${bucket}/${objectKey(src)}`, + ContentType: mimeFor(dst), + MetadataDirective: 'REPLACE', + })); + return true; + } catch (err) { + if (isMissingError(err)) return false; + throw err; + } + } + + async function listAll(listPrefix = '') { + const out = []; + if (!remote) { + const base = listPrefix ? localFile(listPrefix) : dir; + if (!base || !fs.existsSync(base)) return out; + const walk = (abs, rel) => { + for (const name of fs.readdirSync(abs)) { + const childAbs = path.join(abs, name); + const childRel = rel ? `${rel}/${name}` : name; + let st; + try { + st = fs.statSync(childAbs); + } catch { + continue; + } + if (st.isDirectory()) walk(childAbs, childRel); + else out.push({ key: childRel, size: st.size }); + } + }; + walk(base, listPrefix ? normalizeKey(listPrefix) || '' : ''); + return out; + } + let token = null; + const fullPrefix = prefix ? `${prefix}/${listPrefix}` : listPrefix; + do { + const res = await client.send(new ListObjectsV2Command({ + Bucket: bucket, + Prefix: fullPrefix || undefined, + ContinuationToken: token || undefined, + })); + for (const item of res.Contents || []) { + const key = prefix ? String(item.Key).slice(prefix.length + 1) : String(item.Key); + out.push({ key, size: item.Size || 0 }); + } + token = res.IsTruncated ? res.NextContinuationToken : null; + } while (token); + return out; + } + + + + async function localize(key, opts = {}) { + const k = normalizeKey(key); + if (!k) return null; + if (localExists(k)) return localFile(k); + if (!remote) return null; + const cf = cacheFile(k); + if (cf && fs.existsSync(cf)) { + const info = await head(k); + if (info && info.size === fs.statSync(cf).size) return cf; + try { fs.unlinkSync(cf); } catch {} + } + const source = await getStream(k); + if (!source) return null; + fs.mkdirSync(path.dirname(cf), { recursive: true }); + const tmp = `${cf}.${process.pid}.${Date.now()}.tmp`; + await pipeline(source.stream, fs.createWriteStream(tmp)); + if (opts.useCache === false) return tmp; + fs.renameSync(tmp, cf); + return cf; + } + + async function persist(key, localPath) { + const k = normalizeKey(key); + if (!k) return false; + const src = path.resolve(localPath); + if (!fs.existsSync(src)) return false; + if (!remote) return true; + const info = await head(k); + if (info && info.size === fs.statSync(src).size) { + if (!keepLocal) unlinkLocalOnly(k); + return true; + } + await putFile(k, src); + if (!keepLocal) { + try { fs.unlinkSync(src); } catch {} + const cf = cacheFile(k); + if (cf && fs.existsSync(cf)) { + try { fs.unlinkSync(cf); } catch {} + } + } + return true; + } + + async function streamTo(res, key, opts = {}) { + const k = normalizeKey(key); + if (!k) return false; + if (localFallback && localExists(k)) { + const fp = localFile(k); + if (opts.cacheControl) res.setHeader('Cache-Control', opts.cacheControl); + if (opts.download) { + res.download(fp, opts.name || path.basename(fp), opts.callback || (() => {})); + return true; + } + res.setHeader('Content-Type', opts.contentType || mimeFor(k)); + res.sendFile(fp); + return true; + } + const source = await getStream(k); + if (!source) return false; + if (opts.cacheControl) res.setHeader('Cache-Control', opts.cacheControl); + res.setHeader('Content-Type', opts.contentType || source.contentType || mimeFor(k)); + if (source.contentLength) res.setHeader('Content-Length', String(source.contentLength)); + if (opts.download) { + const name = opts.name || path.basename(k); + const ascii = name.replace(/[^\x20-\x7E]/g, '_').replace(/"/g, ''); + res.setHeader('Content-Disposition', `attachment; filename="${ascii}"; filename*=UTF-8''${encodeURIComponent(name)}`); + } + await pipeline(source.stream, res); + return true; + } + + async function downloadAll(destDir, opts = {}) { + fs.mkdirSync(destDir, { recursive: true }); + const root = path.resolve(destDir); + const objects = await listAll(''); + let count = 0; + for (const obj of objects) { + if (!isObjectKey(obj.key)) continue; + const k = normalizeKey(obj.key); + if (!k) continue; + const target = path.resolve(root, ...k.split('/')); + if (!target.startsWith(root + path.sep)) continue; + const source = await getStream(k); + if (!source) continue; + fs.mkdirSync(path.dirname(target), { recursive: true }); + await pipeline(source.stream, fs.createWriteStream(target)); + count++; + } + if (opts.pruneThumbs !== false) { + const thumbs = path.join(root, '.thumbs'); + if (fs.existsSync(thumbs)) fs.rmSync(thumbs, { recursive: true, force: true }); + } + return count; + } + + async function uploadTree(srcDir) { + const root = path.resolve(srcDir); + if (!fs.existsSync(root)) return 0; + let count = 0; + const walk = async (abs, rel) => { + for (const name of fs.readdirSync(abs)) { + const childAbs = path.join(abs, name); + const childRel = rel ? `${rel}/${name}` : name; + let st; + try { + st = fs.statSync(childAbs); + } catch { + continue; + } + if (st.isDirectory()) { + if (name === '.thumbs' || name === '.cache') continue; + await walk(childAbs, childRel); + continue; + } + const k = normalizeKey(childRel); + if (!k) continue; + await putFile(k, childAbs); + count++; + } + }; + await walk(root, ''); + return count; + } + + async function ensureBucket() { + if (!remote || !bucket) return false; + try { + await client.send(new HeadBucketCommand({ Bucket: bucket })); + return true; + } catch (err) { + const status = err && err.$metadata && err.$metadata.httpStatusCode; + const name = err && (err.name || err.Code); + if (status && status !== 404 && name !== 'NotFound' && name !== 'NoSuchBucket') throw err; + } + try { + await client.send(new CreateBucketCommand({ Bucket: bucket })); + return true; + } catch (err) { + const name = err && (err.name || err.Code); + if (name === 'BucketAlreadyOwnedByYou' || name === 'BucketAlreadyExists') return true; + throw err; + } + } + + async function usage() { + const objects = (await listAll('')).filter(o => isObjectKey(o.key)); + return { + count: objects.length, + size_bytes: objects.reduce((sum, o) => sum + (o.size || 0), 0), + bucket: remote ? bucket : null, + prefix: prefix || null, + }; + } + + function pruneCache(now = Date.now()) { + if (!cacheMaxAgeMs || !fs.existsSync(cacheDir)) return 0; + let removed = 0; + const walk = (abs) => { + for (const name of fs.readdirSync(abs)) { + const childAbs = path.join(abs, name); + let st; + try { + st = fs.statSync(childAbs); + } catch { + continue; + } + if (st.isDirectory()) { + walk(childAbs); + continue; + } + if (now - st.mtimeMs > cacheMaxAgeMs) { + try { fs.unlinkSync(childAbs); removed++; } catch {} + } + } + }; + walk(cacheDir); + return removed; + } + + return { + bucket, + dir, + cacheDir, + isRemote, + keyFromPath, + publicPath, + localFile, + localExists, + unlinkLocalOnly, + normalizeKey, + put, + putFile, + head, + exists, + sizeOf, + getStream, + getBuffer, + del, + copyObject, + listAll, + localize, + persist, + streamTo, + downloadAll, + uploadTree, + ensureBucket, + usage, + pruneCache, + }; +} + +module.exports = { createStorage, mimeFor, normalizeKey }; + diff --git a/worker.js b/worker.js index 760f087..a409255 100644 --- a/worker.js +++ b/worker.js @@ -9,7 +9,7 @@ const crypto = require('crypto'); const PHOTO_MAX_ATTEMPTS = 3; const PHOTO_AI_TIMEOUT_MS = 300000; -function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntries, sharp, photoAiUrl, uploadsDir, originalsDir }) { +function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntries, sharp, photoAiUrl, uploadsDir, originalsDir, storage }) { const AI_URL = photoAiUrl || process.env.PHOTO_AI_URL || ''; const IDLE_MIN = 2000; const IDLE_MAX = 30000; @@ -71,30 +71,7 @@ function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntrie } } - function safeUnlinkPath(p) { - try { - if (!p) return; - const abs = path.resolve(p); - const root = path.resolve(uploadsDir); - if (abs !== root && abs.startsWith(root + path.sep)) fs.unlinkSync(abs); - } catch (e) { console.error('photo worker unlink:', e.message); } - } - - function backupOldFile(oldRelPath) { - if (!oldRelPath) return null; - const oldAbs = path.join(uploadsDir, String(oldRelPath).replace(/^\/+/, '').replace(/^uploads\//, '')); - if (!fs.existsSync(oldAbs)) return null; - const backupName = crypto.randomBytes(12).toString('hex') + (path.extname(oldAbs) || '.jpg'); - const backupPath = path.join(originalsDir, backupName); - try { - fs.renameSync(oldAbs, backupPath); - return `/uploads/.originals/${backupName}`; - } catch (e) { - console.error('photo worker backup failed:', e.message); - return null; - } - } - async function enhanceWithSharp(srcAbs, params) { + async function enhanceWithSharp(srcKey, params) { if (!sharp) throw new Error('sharp недоступен на сервере'); const p = params || {}; const clamp = (v, min, max, def) => { @@ -106,7 +83,9 @@ function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntrie const saturate = clamp(p.saturate, 0, 300, 100); const sharpAmt = clamp(p.sharp, 0, 100, 0); const denoise = clamp(p.denoise, 0, 100, 0); - let pipeline = sharp(srcAbs).rotate(); + const srcPath = await storage.localize(srcKey); + if (!srcPath) throw new Error('Файл фото не найден'); + let pipeline = sharp(srcPath).rotate(); if (denoise > 0) pipeline = pipeline.median(denoise > 70 ? 5 : 3); pipeline = pipeline.modulate({ brightness: brightness / 100, saturation: saturate / 100 }); if (contrast !== 100) { @@ -117,13 +96,16 @@ function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntrie pipeline = pipeline.sharpen({ sigma: 0.5 + (sharpAmt / 100) * 1.5, m1: 0, m2: 1 + sharpAmt / 50 }); } const newName = crypto.randomBytes(12).toString('hex') + '.jpg'; - await pipeline.jpeg({ quality: 92, mozjpeg: true }).toFile(path.join(uploadsDir, newName)); + const outPath = path.join(uploadsDir, newName); + await pipeline.jpeg({ quality: 92, mozjpeg: true }).toFile(outPath); + await storage.persist(newName, outPath); return `/uploads/${newName}`; } - async function runAiEnhance(srcAbs) { + async function runAiEnhance(srcKey) { if (!AI_URL) throw new Error('PHOTO_AI_URL не настроен'); - const buf = fs.readFileSync(srcAbs); + const buf = await storage.getBuffer(srcKey); + if (!buf) throw new Error('Файл фото не найден'); const fd = new FormData(); fd.append('image', new Blob([buf], { type: 'image/jpeg' }), 'photo.jpg'); fd.append('scale', '2'); @@ -135,7 +117,9 @@ function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntrie if (!resp.ok) throw new Error('AI service error: ' + resp.status); const out = Buffer.from(await resp.arrayBuffer()); const newName = crypto.randomBytes(12).toString('hex') + '.jpg'; - fs.writeFileSync(path.join(uploadsDir, newName), out); + const outPath = path.join(uploadsDir, newName); + fs.writeFileSync(outPath, out); + await storage.persist(newName, outPath); return `/uploads/${newName}`; } async function applyResult(job, newPath) { @@ -159,9 +143,8 @@ function createPhotoEnhanceWorker({ pool, getSetting, logAudit, invalidateEntrie return true; } const photoPath = rows[0].photo_path; - const srcAbs = path.join(uploadsDir, String(photoPath).replace(/^\/+/, '').replace(/^uploads\//, '')); try { - const newPath = job.action === 'ai' ? await runAiEnhance(srcAbs) : await enhanceWithSharp(srcAbs, job.params); + const newPath = job.action === 'ai' ? await runAiEnhance(photoPath) : await enhanceWithSharp(photoPath, job.params); await applyResult(job, newPath); stats.done++; stats.last_at = new Date().toISOString();