Files
hop.noname.company/apps/server/src/storage.ts

651 lines
25 KiB
TypeScript

import { createHash, randomUUID } from 'node:crypto';
import { mkdirSync, statfsSync, existsSync } from 'node:fs';
import { createWriteStream, createReadStream } from 'node:fs';
import { rename, unlink, readdir, stat, open } from 'node:fs/promises';
import { resolve } from 'node:path';
import { Transform, Writable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { and, eq, gt, isNull, inArray, sql, or } from 'drizzle-orm';
import sharp from 'sharp';
import type { FastifyRequest } from 'fastify';
import type { Guest, Service } from './service.js';
import * as s from './db/schema.js';
import { ApiError, fail } from './security.js';
export type StorageSubject = { id: string; kind?: 'user'; sessionId?: string };
export class Storage {
private active = new Map<
string,
{ guestId: string; controller: AbortController; fileId: string; sessionId?: string }
>();
private cleaning = false;
private cleanupPromise: Promise<void> | undefined;
readonly temp: string;
readonly originals: string;
constructor(readonly service: Service) {
this.temp = resolve(service.cfg.uploadDir, 'temporary');
this.originals = resolve(service.cfg.uploadDir, 'originals');
mkdirSync(this.temp, { recursive: true, mode: 0o700 });
mkdirSync(this.originals, { recursive: true, mode: 0o700 });
}
private request(g: StorageSubject, id: string) {
return g.kind === 'user'
? this.service.accounts.request(g.id, id)
: this.service.request(g as Guest, id);
}
private access(g: StorageSubject, id: string) {
return g.kind === 'user'
? this.service.accounts.access(g.id, id)
: this.service.accessDetail(g as Guest, id);
}
private blocked(g: StorageSubject, row: ReturnType<Storage['request']>) {
return g.kind === 'user'
? !this.service.accounts.canTransfer(row)
: this.service.isBlocked(g.id, row.recipientGuestId!);
}
private isSender(g: StorageSubject, row: ReturnType<Storage['request']>) {
return g.kind === 'user' ? row.senderUserId === g.id : row.senderGuestId === g.id;
}
fileRows(id: string) {
return this.service.db.select().from(s.files).where(eq(s.files.requestId, id)).all();
}
createManifest(
g: StorageSubject,
requestId: string,
manifest: { name: string; size: number; type: string }[],
) {
for (const [position, file] of manifest.entries())
this.service.db
.insert(s.files)
.values({
id: randomUUID(),
requestId,
ownerGuestId: g.kind === 'user' ? null : g.id,
ownerUserId: g.kind === 'user' ? g.id : null,
position,
name: file.name,
size: file.size,
type: file.type,
storageKey: randomUUID(),
status: 'pending',
updatedAt: Date.now(),
})
.run();
}
private used(guestId?: string) {
return this.service.db
.select({ bytes: sql<number>`coalesce(sum(${s.files.size}),0)` })
.from(s.files)
.where(
and(
inArray(s.files.status, ['stored', 'cleaning']),
sql`${s.files.ownerGuestId} IS NOT NULL`,
guestId ? eq(s.files.ownerGuestId, guestId) : undefined,
),
)
.get()!.bytes;
}
private reserved(guestId?: string) {
return this.service.db
.select({ bytes: sql<number>`coalesce(sum(${s.reservations.bytes}),0)` })
.from(s.reservations)
.where(
and(
gt(s.reservations.expiresAt, Date.now()),
sql`${s.reservations.guestId} IS NOT NULL`,
guestId ? eq(s.reservations.guestId, guestId) : undefined,
),
)
.get()!.bytes;
}
reserve(g: StorageSubject, requestId: string) {
const { db, cfg } = this.service;
const files = this.fileRows(requestId).filter((f) => f.status !== 'stored');
const missing = files.filter(
(file) =>
!db
.select()
.from(s.reservations)
.where(and(eq(s.reservations.fileId, file.id), gt(s.reservations.expiresAt, Date.now())))
.get(),
);
const total = missing.reduce((sum, file) => sum + file.size, 0);
if (g.kind === 'user') {
const reserved = db
.select({ bytes: sql<number>`coalesce(sum(${s.reservations.bytes}),0)` })
.from(s.reservations)
.where(and(eq(s.reservations.userId, g.id), gt(s.reservations.expiresAt, Date.now())))
.get()!.bytes;
this.service.accounts.ensureQuota(g.id, reserved + total);
}
if (g.kind !== 'user' && this.used(g.id) + this.reserved(g.id) + total > cfg.guestQuota)
fail(413, 'GUEST_QUOTA', '이번 기기의 저장 용량을 초과했습니다.');
if (g.kind !== 'user' && this.used() + this.reserved() + total > cfg.totalGuestQuota)
fail(507, 'TOTAL_QUOTA', '현재 서버의 익명 저장 공간이 부족합니다.');
const allReserved = db
.select({ bytes: sql<number>`coalesce(sum(${s.reservations.bytes}),0)` })
.from(s.reservations)
.where(gt(s.reservations.expiresAt, Date.now()))
.get()!.bytes;
const disk = statfsSync(this.originals, { bigint: true });
if (BigInt(allReserved + total + cfg.minFreeBytes) > disk.bavail * disk.bsize)
fail(507, 'DISK_FULL', '서버의 여유 공간이 부족합니다.');
for (const file of missing)
db.insert(s.reservations)
.values({
id: randomUUID(),
fileId: file.id,
guestId: g.kind === 'user' ? null : g.id,
userId: g.kind === 'user' ? g.id : null,
bytes: file.size,
expiresAt: Date.now() + cfg.orphanHours * 3600000,
})
.onConflictDoUpdate({
target: s.reservations.fileId,
set: { expiresAt: Date.now() + cfg.orphanHours * 3600000 },
})
.run();
}
release(requestId: string) {
this.service.resumable?.abort(requestId);
const db = this.service.db;
for (const file of this.fileRows(requestId)) {
for (const [uploadId, upload] of this.active)
if (upload.fileId === file.id) {
upload.controller.abort();
this.active.delete(uploadId);
}
db.delete(s.reservations).where(eq(s.reservations.fileId, file.id)).run();
if (file.status !== 'stored')
db.update(s.files)
.set({ status: 'failed', updatedAt: Date.now() })
.where(eq(s.files.id, file.id))
.run();
}
}
uploadManifest(g: StorageSubject, requestId: string, position: number) {
if (g.kind === 'user') {
const session = g.sessionId
? this.service.db
.select()
.from(s.accountSessions)
.where(eq(s.accountSessions.id, g.sessionId))
.get()
: null;
if (
!session ||
session.userId !== g.id ||
session.revokedAt ||
session.expiresAt <= Date.now()
)
fail(401, 'SESSION_REVOKED', '계정 세션이 종료되었습니다.');
} else {
const guest = this.service.db.select().from(s.guests).where(eq(s.guests.id, g.id)).get();
if (!guest || guest.revokedAt || guest.expiresAt <= Date.now())
fail(401, 'SESSION_EXPIRED', '기기 세션이 종료되었습니다.');
}
const row = this.request(g, requestId);
if (row.kind !== 'file' || !this.isSender(g, row))
fail(403, 'UPLOAD_FORBIDDEN', '발신자만 업로드할 수 있습니다.');
if (this.service.vault.source(requestId))
fail(409, 'VAULT_TRANSFER_MANAGED', '이미 저장된 파일을 전달합니다.');
if (
row.status !== 'accepted' ||
!row.startBy ||
row.startBy <= Date.now() ||
this.blocked(g, row)
)
fail(409, 'ACCEPTANCE_REQUIRED', '수락된 유효한 요청이 필요합니다.');
const file = this.fileRows(requestId).find((f) => f.position === position);
if (!file) fail(404, 'FILE_NOT_REQUESTED', '요청에 없는 파일입니다.');
if (file!.status === 'uploading') fail(409, 'UPLOAD_ACTIVE', '파일이 처리 중입니다.');
return file!;
}
async upload(
g: StorageSubject,
requestId: string,
position: number,
request: FastifyRequest,
key: string,
stagedPath?: string,
) {
const { db, cfg } = this.service;
let file: typeof s.files.$inferSelect;
let uploadId: string;
const existing = this.fileRows(requestId).find((f) => f.position === position);
if (existing?.status === 'stored') {
const row = this.request(g, requestId);
if (!this.isSender(g, row)) fail(403, 'UPLOAD_FORBIDDEN', '발신자만 업로드할 수 있습니다.');
if (existing.uploadKey !== key) fail(409, 'FILE_STORED', '이미 저장된 파일입니다.');
const checksum = createHash('sha256');
let bytes = 0,
count = 0;
for await (const part of request.parts({
limits: { files: 1, fields: 0, parts: 1, fileSize: cfg.maxFileBytes + 1 },
})) {
if (part.type !== 'file')
throw new ApiError(400, 'MULTIPART_INVALID', '파일 한 개가 필요합니다.');
count++;
if (part.filename !== existing.name)
fail(409, 'IDEMPOTENCY_CONFLICT', '같은 키의 파일 내용이 다릅니다.');
for await (const chunk of part.file) {
bytes += chunk.length;
if (bytes > existing.size)
fail(409, 'IDEMPOTENCY_CONFLICT', '같은 키의 파일 내용이 다릅니다.');
checksum.update(chunk);
}
}
if (count !== 1 || bytes !== existing.size || checksum.digest('hex') !== existing.checksum)
fail(409, 'IDEMPOTENCY_CONFLICT', '같은 키의 파일 내용이 다릅니다.');
return { uploadId: existing.uploadId!, fileId: existing.id, status: 'stored' };
}
db.transaction(() => {
const row = this.request(g, requestId);
if (row.kind !== 'file' || !this.isSender(g, row))
fail(403, 'UPLOAD_FORBIDDEN', '이 요청에 파일을 올릴 수 없습니다.');
if (
row.status !== 'accepted' ||
!row.startBy ||
row.startBy <= Date.now() ||
this.blocked(g, row)
)
fail(409, 'ACCEPTANCE_REQUIRED', '수락된 유효한 요청이 필요합니다.');
file = this.fileRows(requestId).find((f) => f.position === position)!;
if (!file) fail(400, 'FILE_NOT_REQUESTED', '요청에 없는 파일입니다.');
if (file.status === 'stored') fail(409, 'FILE_STORED', '이미 저장된 파일입니다.');
if (
file.status === 'uploading' ||
[...this.active.values()].filter((v) => v.guestId === `${g.kind ?? 'guest'}:${g.id}`)
.length >= (g.kind === 'user' ? 2 : 1)
)
fail(409, 'UPLOAD_ACTIVE', '진행 중인 업로드를 먼저 끝내 주세요.');
this.reserve(g, requestId);
uploadId = randomUUID();
db.update(s.files)
.set({ status: 'uploading', uploadId, uploadKey: key, updatedAt: Date.now() })
.where(eq(s.files.id, file.id))
.run();
db.update(s.batches)
.set({ status: 'uploading' })
.where(eq(s.batches.requestId, requestId))
.run();
});
const controller = new AbortController(),
tempPath = stagedPath ?? resolve(this.temp, uploadId!);
this.active.set(uploadId!, {
guestId: `${g.kind ?? 'guest'}:${g.id}`,
controller,
sessionId: g.sessionId,
fileId: file!.id,
});
request.raw.once('aborted', () => controller.abort());
this.service.notifyTransfer(requestId);
try {
let parts = 0;
let bytes = 0;
const checksum = createHash('sha256');
for await (const part of request.parts({
limits: { files: 1, fields: 0, parts: 1, fileSize: cfg.maxFileBytes + 1 },
})) {
if (part.type !== 'file')
throw new ApiError(400, 'MULTIPART_INVALID', '파일 한 개만 업로드해 주세요.');
parts++;
if (part.filename !== file!.name)
fail(409, 'METADATA_MISMATCH', '요청한 파일 이름과 다릅니다.');
const counter = new Transform({
transform(chunk, _encoding, callback) {
bytes += chunk.length;
if (bytes > file!.size || bytes > cfg.maxFileBytes)
return callback(
new ApiError(413, 'FILE_SIZE_MISMATCH', '파일 크기가 요청과 다릅니다.'),
);
checksum.update(chunk);
callback(null, chunk);
},
});
await pipeline(
part.file,
counter,
stagedPath
? new Writable({
write(_chunk, _encoding, callback) {
callback();
},
})
: createWriteStream(tempPath, { flags: 'wx', mode: 0o600 }),
{ signal: controller.signal },
);
if (part.file.truncated || bytes !== file!.size)
fail(409, 'FILE_SIZE_MISMATCH', '파일 크기가 요청과 다릅니다.');
}
if (parts !== 1) fail(400, 'MULTIPART_INVALID', '파일 한 개가 필요합니다.');
const originalPath = resolve(this.originals, file!.storageKey);
await rename(tempPath, originalPath);
try {
db.transaction(() => {
if (g.kind === 'user') {
const session = g.sessionId
? db
.select()
.from(s.accountSessions)
.where(eq(s.accountSessions.id, g.sessionId))
.get()
: null;
if (
!session ||
session.userId !== g.id ||
session.revokedAt ||
session.expiresAt <= Date.now()
)
fail(401, 'SESSION_REVOKED', '계정 세션이 종료되었습니다.');
}
const row = this.request(g, requestId),
current = db.select().from(s.files).where(eq(s.files.id, file!.id)).get();
if (
row.status !== 'accepted' ||
this.blocked(g, row) ||
current?.uploadId !== uploadId ||
current.status !== 'uploading'
)
fail(409, 'UPLOAD_CANCELLED', '전송이 취소되었거나 권한이 변경되었습니다.');
const batch = db
.select()
.from(s.batches)
.where(eq(s.batches.requestId, requestId))
.get()!,
now = Date.now(),
itemId = randomUUID();
db.update(s.files)
.set({ status: 'stored', checksum: checksum.digest('hex'), updatedAt: now })
.where(eq(s.files.id, file!.id))
.run();
db.delete(s.reservations).where(eq(s.reservations.fileId, file!.id)).run();
db.insert(s.items)
.values({
id: itemId,
batchId: batch.id,
kind: 'file',
fileId: file!.id,
createdAt: now,
expiresAt: now + cfg.fileRetentionDays * 86400000,
})
.run();
this.service.accounts.createAccess(row, itemId);
const all = this.fileRows(requestId);
db.update(s.batches)
.set({
status: all.every((f) => f.status === 'stored') ? 'stored' : 'waiting',
storedAt: now,
})
.where(eq(s.batches.id, batch.id))
.run();
});
} catch (error) {
await unlink(originalPath).catch(() => {});
throw error;
}
return { uploadId: uploadId!, fileId: file!.id, status: 'stored' };
} catch (error) {
await unlink(tempPath).catch(() => {});
db.delete(s.reservations).where(eq(s.reservations.fileId, file!.id)).run();
db.update(s.files)
.set({ status: 'failed', updatedAt: Date.now() })
.where(and(eq(s.files.id, file!.id), eq(s.files.uploadId, uploadId!)))
.run();
db.update(s.batches)
.set({ status: 'waiting' })
.where(and(eq(s.batches.requestId, requestId), eq(s.batches.status, 'uploading')))
.run();
if (error instanceof ApiError) throw error;
if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOSPC')
fail(507, 'DISK_FULL', '서버 저장 공간이 부족합니다.');
fail(400, 'UPLOAD_FAILED', '업로드가 중단되었습니다. 파일을 다시 선택해 재시도해 주세요.');
} finally {
this.active.delete(uploadId!);
this.service.notifyTransfer(requestId);
}
}
cancel(g: StorageSubject, id: string) {
const file = this.service.db
.select()
.from(s.files)
.where(
and(
eq(s.files.uploadId, id),
g.kind === 'user' ? eq(s.files.ownerUserId, g.id) : eq(s.files.ownerGuestId, g.id),
),
)
.get();
if (!file) fail(404, 'UPLOAD_NOT_FOUND', '업로드를 찾을 수 없습니다.');
if (file!.status !== 'uploading')
fail(409, 'UPLOAD_NOT_ACTIVE', '진행 중인 업로드만 취소할 수 있습니다.');
this.active.get(id)?.controller.abort();
this.service.db
.update(s.files)
.set({ status: 'failed', updatedAt: Date.now() })
.where(eq(s.files.id, file!.id))
.run();
}
download(g: StorageSubject, accessId: string) {
const row = this.access(g, accessId);
if (row.item.kind !== 'file' || !row.item.fileId)
fail(404, 'FILE_NOT_FOUND', '파일을 찾을 수 없습니다.');
const file = this.service.db
.select()
.from(s.files)
.where(and(eq(s.files.id, row.item.fileId!), eq(s.files.status, 'stored')))
.get();
if (!file || !existsSync(resolve(this.originals, file.storageKey)))
fail(404, 'FILE_NOT_FOUND', '파일을 찾을 수 없습니다.');
return { file: file!, path: resolve(this.originals, file!.storageKey) };
}
async preview(g: StorageSubject, id: string) {
const { path } = this.download(g, id);
try {
const handle = await open(path, 'r');
const magic = Buffer.alloc(16);
try {
await handle.read(magic, 0, 16, 0);
} finally {
await handle.close();
}
const raster =
magic.subarray(0, 3).equals(Buffer.from([255, 216, 255])) ||
magic.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) ||
/^GIF8[79]a/.test(magic.toString('ascii')) ||
(magic.toString('ascii', 0, 4) === 'RIFF' && magic.toString('ascii', 8, 12) === 'WEBP') ||
(magic.toString('ascii', 4, 8) === 'ftyp' &&
['avif', 'avis', 'heic', 'heix', 'mif1'].includes(magic.toString('ascii', 8, 12)));
if (!raster) throw Error();
const image = sharp(path, { limitInputPixels: 40000000, animated: false });
const metadata = await image.metadata();
if (!['jpeg', 'png', 'webp', 'gif', 'avif', 'heif'].includes(metadata.format ?? ''))
throw Error();
return await image
.resize(720, 720, { fit: 'inside', withoutEnlargement: true })
.rotate()
.webp({ quality: 78 })
.toBuffer();
} catch {
fail(415, 'PREVIEW_UNSUPPORTED', '이 파일은 이미지 미리보기를 지원하지 않습니다.');
}
}
deleteAccess(g: StorageSubject, ids: string[]) {
const { db, cfg } = this.service;
return db.transaction(() => {
const unique = [...new Set(ids)],
rows = unique.map((id) => this.access(g, id));
const id = randomUUID(),
now = Date.now();
db.insert(s.deletions)
.values({
id,
guestId: g.kind === 'user' ? null : g.id,
userId: g.kind === 'user' ? g.id : null,
createdAt: now,
undoUntil: now + cfg.undoSeconds * 1000,
})
.run();
for (const row of rows) {
db.update(s.access).set({ deletedAt: now }).where(eq(s.access.id, row.access.id)).run();
db.insert(s.deletionTargets)
.values({ id: randomUUID(), operationId: id, accessId: row.access.id })
.run();
}
return { operationId: id, undoUntil: now + cfg.undoSeconds * 1000 };
});
}
undo(g: StorageSubject, id: string) {
const { db } = this.service;
return db.transaction(() => {
const operation = db
.select()
.from(s.deletions)
.where(
and(
eq(s.deletions.id, id),
g.kind === 'user' ? eq(s.deletions.userId, g.id) : eq(s.deletions.guestId, g.id),
),
)
.get();
if (!operation) fail(404, 'DELETE_NOT_FOUND', '삭제 작업을 찾을 수 없습니다.');
if (operation!.restoredAt) return { restored: true };
if (operation!.undoUntil <= Date.now())
fail(409, 'UNDO_EXPIRED', '실행 취소 시간이 지났습니다.');
const targets = db
.select()
.from(s.deletionTargets)
.where(eq(s.deletionTargets.operationId, id))
.all();
let count = 0;
for (const target of targets) {
const row = db
.select({ access: s.access, item: s.items })
.from(s.access)
.innerJoin(s.items, eq(s.access.itemId, s.items.id))
.where(
and(
eq(s.access.id, target.accessId),
g.kind === 'user' ? eq(s.access.userId, g.id) : eq(s.access.guestId, g.id),
eq(s.access.deletedAt, operation!.createdAt),
gt(s.items.expiresAt, Date.now()),
),
)
.get();
if (row) {
if (g.kind === 'user' && row.item.fileId) {
const file = db.select().from(s.files).where(eq(s.files.id, row.item.fileId)).get();
if (file && !this.service.accounts.canStoreFile(g.id, file))
fail(413, 'ACCOUNT_QUOTA', '보관함 용량이 부족해 복원하지 못했습니다.');
}
db.update(s.access).set({ deletedAt: null }).where(eq(s.access.id, row.access.id)).run();
count++;
}
}
db.update(s.deletions).set({ restoredAt: Date.now() }).where(eq(s.deletions.id, id)).run();
return { restored: count > 0 };
});
}
cleanup(): Promise<void> {
if (this.cleanupPromise) return this.cleanupPromise;
this.cleanupPromise = this.performCleanup().finally(() => {
this.cleanupPromise = undefined;
});
return this.cleanupPromise;
}
private async performCleanup() {
if (this.cleaning) return;
this.cleaning = true;
try {
this.service.resumable?.cleanup();
const { db, cfg } = this.service,
now = Date.now();
const expired = db
.select()
.from(s.reservations)
.where(sql`${s.reservations.expiresAt} <= ${now}`)
.all();
for (const r of expired) {
const file = db.select().from(s.files).where(eq(s.files.id, r.fileId)).get();
if (file) this.release(file.requestId);
}
for (const file of db.select().from(s.files).where(eq(s.files.status, 'uploading')).all()) {
if (!file.uploadId || !this.active.has(file.uploadId)) {
db.update(s.files)
.set({ status: 'failed', updatedAt: now })
.where(eq(s.files.id, file.id))
.run();
}
}
const all = db
.select()
.from(s.files)
.where(inArray(s.files.status, ['stored', 'cleaning']))
.all();
for (const file of all) {
const refs = db
.select({ id: s.items.id })
.from(s.items)
.innerJoin(s.access, eq(s.access.itemId, s.items.id))
.where(
and(
eq(s.items.fileId, file.id),
gt(s.items.expiresAt, now),
orLive(now, cfg.undoSeconds),
),
)
.all();
if (refs.length) continue;
db.update(s.files).set({ status: 'cleaning' }).where(eq(s.files.id, file.id)).run();
try {
await unlink(resolve(this.originals, file.storageKey));
} catch (error) {
if (!error || typeof error !== 'object' || !('code' in error) || error.code !== 'ENOENT')
continue;
}
db.update(s.files)
.set({ status: 'failed', checksum: null, updatedAt: now })
.where(eq(s.files.id, file.id))
.run();
}
const validKeys = new Set(
db
.select()
.from(s.files)
.where(inArray(s.files.status, ['stored', 'cleaning']))
.all()
.map((f) => f.storageKey),
);
for (const name of await readdir(this.originals)) {
if (validKeys.has(name)) continue;
const path = resolve(this.originals, name),
metadata = await stat(path);
if (metadata.mtimeMs < now - cfg.orphanHours * 3600000) await unlink(path).catch(() => {});
}
for (const name of await readdir(this.temp)) {
if (this.active.has(name)) continue;
const path = resolve(this.temp, name);
const metadata = await stat(path);
if (metadata.mtimeMs < now - cfg.orphanHours * 3600000) await unlink(path).catch(() => {});
}
} finally {
this.cleaning = false;
}
}
abortSession(id: string) {
this.service.resumable?.abortSession(id);
for (const upload of this.active.values())
if (upload.sessionId === id) upload.controller.abort();
}
abortUser(id: string) {
this.service.resumable?.abortUser(id);
for (const upload of this.active.values())
if (upload.guestId === `user:${id}`) upload.controller.abort();
}
close() {
for (const upload of this.active.values()) upload.controller.abort();
}
}
function orLive(now: number, undoSeconds: number) {
return sql`(${s.access.deletedAt} IS NULL OR ${s.access.deletedAt} > ${now - undoSeconds * 1000})`;
}