Separate completed transfers and retain original file extensions

This commit is contained in:
2026-10-14 16:10:59 +09:00
parent 3852c87e2e
commit 69e7449ba7
11 changed files with 736 additions and 31 deletions

View File

@@ -623,7 +623,7 @@ export async function createApp(cfg: Config, options: { mailer?: Mailer } = {})
app.get('/api/access/:id/download', async (req, reply) => {
const accessId = id(req),
g = accessSubject(req, accessId);
const { file, stream } = await service.storage.openDownload(g, accessId);
const { file, stream, access } = await service.storage.openDownload(g, accessId);
const fallback =
file.name.replace(/[^a-zA-Z0-9._ -]/g, '_').replace(/^[. ]+/, '') || 'download';
const encoded = encodeURIComponent(file.name).replace(
@@ -643,6 +643,15 @@ export async function createApp(cfg: Config, options: { mailer?: Mailer } = {})
stream.destroy();
throw error;
}
reply.raw.once('finish', () => {
if (reply.statusCode === 200 && stream.readableEnded && access.role === 'recipient') {
try {
service.storage.layout.downloaded(file.id);
} catch (err) {
req.log.warn({ err }, 'Completed file relocation will be retried');
}
}
});
return reply.send(stream);
});
app.get('/api/access/:id/preview', async (req, reply) =>

View File

@@ -1,19 +1,29 @@
import { statSync } from 'node:fs';
import { resolve } from 'node:path';
import { and, eq, gt, inArray } from 'drizzle-orm';
import * as s from './db/schema.js';
import { fail } from './security.js';
export class FileAvailability {
constructor(service, directory) {
constructor(service, filePath) {
this.service = service;
this.directory = directory;
this.filePath = filePath;
this.pending = new Set();
}
check(file) {
try {
if (statSync(resolve(this.directory, file.storageKey)).isFile()) return true;
if (statSync(this.filePath(file)).isFile()) return true;
} catch (error) {
if (['ENOENT', 'ENOTDIR'].includes(error.code)) {
try {
if (this.service.storage.layout.recover(file)) return this.check(file);
} catch {
fail(
503,
'STORAGE_UNAVAILABLE',
'파일 저장소에 접근할 수 없습니다. 잠시 후 다시 시도해 주세요.',
);
}
}
if (!['ENOENT', 'ENOTDIR'].includes(error.code))
fail(
503,

View File

@@ -0,0 +1,213 @@
import { randomUUID } from 'node:crypto';
import {
existsSync,
mkdirSync,
renameSync,
statSync,
readFileSync,
writeFileSync,
unlinkSync,
} from 'node:fs';
import { basename, extname, resolve } from 'node:path';
import { and, eq, gt, inArray, isNotNull, sql } from 'drizzle-orm';
import * as s from './db/schema.js';
export function fileExtension(name) {
const extension = extname(name).toLowerCase();
return /^\.[a-z0-9]{1,16}$/.test(extension) ? extension : '';
}
export function newStorageKey(name) {
return randomUUID() + fileExtension(name);
}
export class FileLayout {
constructor(service, canMove) {
this.service = service;
this.canMove = canMove;
this.originals = resolve(service.cfg.uploadDir, 'originals');
this.completed = resolve(service.cfg.uploadDir, 'completed');
this.journal = resolve(service.cfg.uploadDir, '.file-moves');
this.pending = new Set();
for (const directory of [this.originals, this.completed, this.journal])
mkdirSync(directory, { recursive: true, mode: 0o700 });
}
path(file) {
const current = this.service.db
.select({ key: s.files.storageKey })
.from(s.files)
.where(eq(s.files.id, file.id))
.get();
return this.keyPath(current?.key ?? file.storageKey);
}
keyPath(key) {
const name = key.startsWith('completed/') ? key.slice('completed/'.length) : key;
if (!name || basename(name) !== name || ['.', '..'].includes(name))
throw Error('Invalid stored file key');
return resolve(key.startsWith('completed/') ? this.completed : this.originals, name);
}
hasAccountReference(id) {
const { db, cfg } = this.service;
return Boolean(
db
.select({ id: s.access.id })
.from(s.items)
.innerJoin(s.access, eq(s.access.itemId, s.items.id))
.where(
and(
eq(s.items.fileId, id),
isNotNull(s.access.userId),
inArray(s.access.role, ['self', 'recipient']),
gt(s.items.expiresAt, Date.now()),
sql`(${s.access.deletedAt} IS NULL OR ${s.access.deletedAt} > ${Date.now() - cfg.undoSeconds * 1000})`,
),
)
.get(),
);
}
recover(file) {
const journalPath = resolve(this.journal, file.id + '.json');
if (!existsSync(journalPath)) return false;
const entry = JSON.parse(readFileSync(journalPath, 'utf8'));
if (entry.fileId !== file.id || entry.checksum !== file.checksum || entry.size !== file.size)
throw Error('Invalid file relocation journal');
const source = this.keyPath(entry.from),
target = this.keyPath(entry.to);
if (![entry.from, entry.to].includes(file.storageKey))
throw Error('File relocation key changed');
if (existsSync(file.storageKey === entry.from ? source : target)) {
unlinkSync(journalPath);
return false;
}
const recoveredKey = file.storageKey === entry.from ? entry.to : entry.from;
const recoveredPath = this.keyPath(recoveredKey);
if (existsSync(recoveredPath)) {
const metadata = statSync(recoveredPath);
if (!metadata.isFile() || metadata.size !== file.size)
throw Error('Incomplete file relocation');
this.service.db
.update(s.files)
.set({ storageKey: recoveredKey })
.where(and(eq(s.files.id, file.id), eq(s.files.storageKey, file.storageKey)))
.run();
unlinkSync(journalPath);
return true;
}
unlinkSync(journalPath);
return false;
}
move(file, key) {
if (key === file.storageKey || !this.canMove(file.id)) return;
const destination = key;
if (existsSync(this.keyPath(destination)))
throw Error('File relocation destination already exists');
const source = this.keyPath(file.storageKey),
target = this.keyPath(destination);
let moved = false;
const journalPath = resolve(this.journal, file.id + '.json');
writeFileSync(
journalPath,
JSON.stringify({
fileId: file.id,
checksum: file.checksum,
size: file.size,
from: file.storageKey,
to: destination,
}),
{ mode: 0o600, flag: 'wx', flush: true },
);
try {
this.service.db.transaction(() => {
renameSync(source, target);
moved = true;
const updated = this.service.db
.update(s.files)
.set({ storageKey: destination })
.where(
and(
eq(s.files.id, file.id),
eq(s.files.storageKey, file.storageKey),
inArray(s.files.status, ['stored', 'cleaning']),
),
)
.run();
if (!updated.changes) throw Error('Stored file changed during relocation');
});
} catch (error) {
if (moved && !existsSync(source)) {
try {
renameSync(target, source);
} catch {}
}
if (existsSync(source)) {
try {
unlinkSync(journalPath);
} catch {}
}
throw error;
}
// A journal cleanup failure must never roll back a committed database/file move.
try {
unlinkSync(journalPath);
} catch {}
}
prepare() {
for (const file of this.service.db
.select()
.from(s.files)
.where(inArray(s.files.status, ['stored', 'cleaning']))
.all()) {
try {
this.recover(file);
const current = this.service.db.select().from(s.files).where(eq(s.files.id, file.id)).get();
const name = basename(current.storageKey),
extension = fileExtension(current.name);
if (extension && !extname(name) && existsSync(this.keyPath(current.storageKey)))
this.move(
current,
(current.storageKey.startsWith('completed/') ? 'completed/' : '') + name + extension,
);
} catch {
// Naming is cosmetic: keep the old key and let availability checks report storage errors.
}
}
}
downloaded(id) {
this.pending.add(id);
this.flush(id);
}
flush(id) {
if (!this.pending.has(id) || !this.canMove(id)) return;
const file = this.service.db.select().from(s.files).where(eq(s.files.id, id)).get();
if (!file || file.status !== 'stored') {
this.pending.delete(id);
return;
}
const directory = this.hasAccountReference(id) ? '' : 'completed/';
this.move(file, directory + basename(file.storageKey));
this.pending.delete(id);
}
reconcile() {
for (const file of this.service.db
.selectDistinct({ id: s.files.id })
.from(s.files)
.innerJoin(s.items, eq(s.items.fileId, s.files.id))
.innerJoin(s.access, eq(s.access.itemId, s.items.id))
.where(
and(
eq(s.files.status, 'stored'),
sql`${s.files.storageKey} LIKE 'completed/%'`,
isNotNull(s.access.userId),
inArray(s.access.role, ['self', 'recipient']),
gt(s.items.expiresAt, Date.now()),
sql`(${s.access.deletedAt} IS NULL OR ${s.access.deletedAt} > ${Date.now() - this.service.cfg.undoSeconds * 1000})`,
),
)
.all())
this.pending.add(file.id);
for (const id of this.pending) {
try {
this.flush(id);
} catch {}
}
}
}

View File

@@ -12,6 +12,7 @@ import type { Guest, Service } from './service.js';
import * as s from './db/schema.js';
import { ApiError, fail } from './security.js';
import { FileAvailability } from './fileAvailability.js';
import { FileLayout, newStorageKey } from './fileLayout.js';
export type StorageSubject = { id: string; kind?: 'user'; sessionId?: string };
export class Storage {
private active = new Map<
@@ -20,6 +21,8 @@ export class Storage {
>();
private cleaning = false;
private cleanupPromise: Promise<void> | undefined;
private readers = new Map<string, number>();
readonly layout: FileLayout;
readonly temp: string;
readonly originals: string;
readonly availability: FileAvailability;
@@ -28,7 +31,11 @@ export class Storage {
this.originals = resolve(service.cfg.uploadDir, 'originals');
mkdirSync(this.temp, { recursive: true, mode: 0o700 });
mkdirSync(this.originals, { recursive: true, mode: 0o700 });
this.availability = new FileAvailability(service, this.originals);
this.layout = new FileLayout(service, (id: string) => !this.readers.has(id));
this.layout.prepare();
this.availability = new FileAvailability(service, (file: typeof s.files.$inferSelect) =>
this.layout.path(file),
);
}
private request(g: StorageSubject, id: string) {
return g.kind === 'user'
@@ -68,7 +75,7 @@ export class Storage {
name: file.name,
size: file.size,
type: file.type,
storageKey: randomUUID(),
storageKey: newStorageKey(file.name),
status: 'pending',
updatedAt: Date.now(),
})
@@ -434,14 +441,29 @@ export class Storage {
if (row.item.kind !== 'file' || !row.item.fileId)
fail(404, 'FILE_NOT_FOUND', '파일을 찾을 수 없습니다.');
const file = this.availability.require(row.item.fileId!);
return { file, path: resolve(this.originals, file.storageKey) };
return { file, path: this.layout.path(file), access: row.access };
}
private beginRead(id: string) {
this.readers.set(id, (this.readers.get(id) ?? 0) + 1);
}
private endRead(id: string) {
const remaining = (this.readers.get(id) ?? 1) - 1;
if (remaining) this.readers.set(id, remaining);
else this.readers.delete(id);
try {
this.layout.flush(id);
} catch {}
}
async openDownload(g: StorageSubject, accessId: string) {
const { file, path } = this.download(g, accessId);
const { file, path, access } = this.download(g, accessId);
this.beginRead(file.id);
try {
const handle = await open(path, 'r');
return { file, stream: handle.createReadStream() };
const stream = handle.createReadStream();
stream.once('close', () => this.endRead(file.id));
return { file, stream, access };
} catch (error) {
this.endRead(file.id);
if (
error &&
typeof error === 'object' &&
@@ -463,7 +485,8 @@ export class Storage {
}
}
async preview(g: StorageSubject, id: string) {
const { path } = this.download(g, id);
const { file, path } = this.download(g, id);
this.beginRead(file.id);
try {
const handle = await open(path, 'r');
const magic = Buffer.alloc(16);
@@ -491,6 +514,8 @@ export class Storage {
.toBuffer();
} catch {
fail(415, 'PREVIEW_UNSUPPORTED', '이 파일은 이미지 미리보기를 지원하지 않습니다.');
} finally {
this.endRead(file.id);
}
}
deleteAccess(g: StorageSubject, ids: string[]) {
@@ -583,6 +608,7 @@ export class Storage {
this.service.resumable?.cleanup();
const { db, cfg } = this.service,
now = Date.now();
this.layout.reconcile();
const expired = db
.select()
.from(s.reservations)
@@ -622,7 +648,7 @@ export class Storage {
if (refs.length) continue;
db.update(s.files).set({ status: 'cleaning' }).where(eq(s.files.id, file.id)).run();
try {
await unlink(resolve(this.originals, file.storageKey));
await unlink(this.layout.path(file));
} catch (error) {
if (!error || typeof error !== 'object' || !('code' in error) || error.code !== 'ENOENT')
continue;
@@ -640,11 +666,38 @@ export class Storage {
.all()
.map((f) => f.storageKey),
);
for (const name of await readdir(this.originals)) {
if (validKeys.has(name)) continue;
const path = resolve(this.originals, name),
metadata = await stat(path);
if (metadata.mtimeMs < now - cfg.orphanHours * 3600000) await unlink(path).catch(() => {});
for (const [directory, prefix] of [
[this.originals, ''],
[this.layout.completed, 'completed/'],
]) {
for (const name of await readdir(directory)) {
const key = prefix + name;
if (validKeys.has(key)) continue;
const path = resolve(directory, name);
try {
const metadata = await stat(path);
if (!metadata.isFile() || metadata.mtimeMs >= now - cfg.orphanHours * 3600000) continue;
if (
db
.select()
.from(s.files)
.where(
and(eq(s.files.storageKey, key), inArray(s.files.status, ['stored', 'cleaning'])),
)
.get()
)
continue;
await unlink(path).catch(() => {});
} catch (error) {
if (
!error ||
typeof error !== 'object' ||
!('code' in error) ||
!['ENOENT', 'ENOTDIR'].includes(String(error.code))
)
continue;
}
}
}
for (const name of await readdir(this.temp)) {
if (this.active.has(name)) continue;

View File

@@ -0,0 +1,393 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { randomUUID } from 'node:crypto';
import { once } from 'node:events';
import { get } from 'node:http';
import { Readable } from 'node:stream';
import { existsSync, readFileSync, renameSync, writeFileSync, unlinkSync } from 'node:fs';
import { resolve } from 'node:path';
import { and, eq, isNotNull } from 'drizzle-orm';
import { fixture } from './helpers.js';
import * as s from '../src/db/schema.js';
async function login(f, c) {
const id = randomUUID(),
email = `${id}@hop.test`;
f.service.db
.insert(s.users)
.values({
id,
email,
displayName: email,
passwordHash: await f.service.auth.password('12345678'),
status: 'active',
verifiedAt: Date.now(),
createdAt: Date.now(),
})
.run();
const r = await f.call(c, '/auth/login', 'POST', { email, password: '12345678' });
assert.equal(r.statusCode, 200);
c.cookie += '; ' + String(r.headers['set-cookie']).split(';')[0];
return id;
}
async function file(f, a, b, name = '사진.JPG') {
const bytes = Buffer.from('unchanged file bytes');
await f.space(a, b);
const r = await f.call(
a,
'/transfer-requests',
'POST',
{
targetDeviceId: b.deviceId,
kind: 'file',
count: 1,
byteSize: bytes.length,
files: [{ name, size: bytes.length, type: 'application/octet-stream' }],
},
randomUUID(),
);
assert.equal(r.statusCode, 200, r.body);
const id = r.json().requestId;
assert.equal(
(await f.call(b, `/transfer-requests/${id}/accept`, 'POST', {}, randomUUID())).statusCode,
200,
);
const boundary = 'layout-upload',
body = Buffer.concat([
Buffer.from(
`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${name}"\r\nContent-Type: application/octet-stream\r\n\r\n`,
),
bytes,
Buffer.from(`\r\n--${boundary}--\r\n`),
]);
const upload = await f.call(
a,
`/uploads?requestId=${id}&fileIndex=0`,
'POST',
body,
randomUUID(),
{ 'content-type': `multipart/form-data; boundary=${boundary}` },
);
assert.equal(upload.statusCode, 200, upload.body);
const row = f.service.storage.fileRows(id)[0];
const item = (await f.call(b, '/device/items')).json().items.find((i) => i.file?.id === row.id);
return { row, item, bytes, id };
}
function current(f, row) {
return f.service.db.select().from(s.files).where(eq(s.files.id, row.id)).get();
}
async function archived(f, row) {
// The HTTP response can finish before the file descriptor's asynchronous close.
for (let i = 0; i < 100 && !current(f, row).storageKey.startsWith('completed/'); i++)
await new Promise((r) => setTimeout(r, 5));
assert.equal(current(f, row).storageKey, `completed/${row.storageKey}`);
}
test('logged-in sender to anonymous recipient: extension kept, completed after download, seven-day expiry and re-download unchanged', async () => {
const f = await fixture();
try {
const a = await f.create(),
b = await f.create();
await login(f, a);
const value = await file(f, a, b),
row = value.row;
assert.match(row.storageKey, /^[a-f0-9-]{36}\.jpg$/);
assert.equal((await f.call(a, '/vault')).json().items.length, 0);
const expiry = f.service.db
.select()
.from(s.items)
.where(eq(s.items.fileId, row.id))
.get().expiresAt;
assert.equal(existsSync(resolve(f.service.storage.originals, row.storageKey)), true);
const downloaded = await f.call(b, `/access/${value.item.accessId}/download`);
assert.equal(downloaded.statusCode, 200);
assert.deepEqual(downloaded.rawPayload, value.bytes);
await archived(f, row);
const stored = current(f, row);
assert.equal(stored.storageKey, `completed/${row.storageKey}`);
assert.equal(existsSync(resolve(f.service.storage.originals, row.storageKey)), false);
assert.deepEqual(readFileSync(f.service.storage.layout.path(stored)), value.bytes);
assert.equal(stored.checksum, row.checksum);
assert.equal(stored.updatedAt, row.updatedAt);
assert.equal(
f.service.db.select().from(s.items).where(eq(s.items.fileId, row.id)).get().expiresAt,
expiry,
);
assert.deepEqual(
(await f.call(b, `/access/${value.item.accessId}/download`)).rawPayload,
value.bytes,
);
} finally {
await f.close();
}
});
test('anonymous sender to signed-in recipient: received-account reference keeps the file in originals', async () => {
const f = await fixture();
try {
const a = await f.create(),
b = await f.create();
await login(f, b);
const value = await file(f, a, b, 'received.pdf');
const downloaded = await f.call(b, `/access/${value.item.accessId}/download`);
assert.equal(downloaded.statusCode, 200);
await new Promise((r) => setImmediate(r));
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
assert(existsSync(f.service.storage.layout.path(value.row)));
assert.equal((await f.call(b, '/inbox')).json().items.length, 1);
} finally {
await f.close();
}
});
test('a sender read or unauthorized read does not classify a transfer as recipient-completed', async () => {
const f = await fixture();
try {
const a = await f.create(),
b = await f.create(),
c = await f.create();
const value = await file(f, a, b);
assert.equal((await f.call(c, `/access/${value.item.accessId}/download`)).statusCode, 404);
const sent = (await f.call(a, '/device/items')).json().items[0];
assert.equal((await f.call(a, `/access/${sent.accessId}/download`)).statusCode, 200);
await new Promise((r) => setImmediate(r));
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
} finally {
await f.close();
}
});
test('relocation waits for all active readers and preserves concurrent download bytes', async () => {
const f = await fixture();
try {
const a = await f.create(),
b = await f.create();
const value = await file(f, a, b);
const guest = f.service.db
.select()
.from(s.guests)
.where(eq(s.guests.id, f.service.deviceGuest(b.deviceId).guestId))
.get();
const first = await f.service.storage.openDownload(guest, value.item.accessId),
second = await f.service.storage.openDownload(guest, value.item.accessId);
f.service.storage.layout.downloaded(value.row.id);
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
const firstClosed = once(first.stream, 'close');
first.stream.resume();
await firstClosed;
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
const chunks = [];
second.stream.on('data', (chunk) => chunks.push(chunk));
await once(second.stream, 'close');
assert.deepEqual(Buffer.concat(chunks), value.bytes);
assert.equal(current(f, value.row).storageKey, `completed/${value.row.storageKey}`);
} finally {
await f.close();
}
});
test('legacy extension-less files migrate without changing IDs, checksums, content or expiry', async () => {
const f = await fixture();
try {
const a = await f.create(),
b = await f.create();
const value = await file(f, a, b, 'old.png');
const legacy = randomUUID();
renameSync(
f.service.storage.layout.path(value.row),
resolve(f.service.storage.originals, legacy),
);
f.service.db
.update(s.files)
.set({ storageKey: legacy })
.where(eq(s.files.id, value.row.id))
.run();
f.service.storage.layout.prepare();
const stored = current(f, value.row);
assert.equal(stored.storageKey, legacy + '.png');
assert.equal(stored.id, value.row.id);
assert.deepEqual(readFileSync(f.service.storage.layout.path(stored)), value.bytes);
assert.equal(stored.checksum, value.row.checksum);
const before = stored.storageKey;
f.service.storage.layout.prepare();
assert.equal(current(f, value.row).storageKey, before);
} finally {
await f.close();
}
});
test('database relocation failure rolls back the filename and preserves the original bytes', async () => {
const f = await fixture();
try {
const a = await f.create(),
b = await f.create();
const value = await file(f, a, b);
f.sqlite.exec(
"CREATE TRIGGER reject_move BEFORE UPDATE OF storageKey ON files BEGIN SELECT RAISE(ABORT, 'test relocation failure'); END",
);
assert.throws(() =>
f.service.storage.layout.move(value.row, `completed/${value.row.storageKey}`),
);
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
assert.deepEqual(readFileSync(f.service.storage.layout.path(value.row)), value.bytes);
assert.equal(
existsSync(f.service.storage.layout.keyPath(`completed/${value.row.storageKey}`)),
false,
);
assert.equal(
existsSync(resolve(f.service.storage.layout.journal, value.row.id + '.json')),
false,
);
f.sqlite.exec('DROP TRIGGER reject_move');
f.service.storage.layout.downloaded(value.row.id);
assert.equal(current(f, value.row).storageKey, `completed/${value.row.storageKey}`);
} finally {
await f.close();
}
});
test('account deletion undo remains protected; restoring an account reference returns an archive to originals', async () => {
const f = await fixture();
try {
const a = await f.create(),
b = await f.create();
await login(f, b);
const value = await file(f, a, b);
const reference = f.service.db
.select()
.from(s.access)
.where(and(eq(s.access.itemId, value.item.id), isNotNull(s.access.userId)))
.get();
assert(reference);
f.service.db
.update(s.access)
.set({ deletedAt: Date.now() })
.where(eq(s.access.id, reference.id))
.run();
f.service.storage.layout.downloaded(value.row.id);
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
f.service.db
.update(s.access)
.set({ deletedAt: Date.now() - f.cfg.undoSeconds * 1000 - 1 })
.where(eq(s.access.id, reference.id))
.run();
f.service.storage.layout.downloaded(value.row.id);
assert.equal(current(f, value.row).storageKey, `completed/${value.row.storageKey}`);
f.service.db
.update(s.access)
.set({ deletedAt: null })
.where(eq(s.access.id, reference.id))
.run();
f.service.storage.layout.reconcile();
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
assert.deepEqual(readFileSync(f.service.storage.layout.path(value.row)), value.bytes);
} finally {
await f.close();
}
});
test('aborting an HTTP download before all bytes arrive leaves the file available in originals', async () => {
const f = await fixture();
try {
const a = await f.create(),
b = await f.create();
const value = await file(f, a, b);
const openDownload = f.service.storage.openDownload.bind(f.service.storage);
let slow;
f.service.storage.openDownload = async (...args) => {
const result = await openDownload(...args);
const closed = once(result.stream, 'close');
result.stream.destroy();
await closed;
slow = Readable.from(
(async function* () {
yield value.bytes.subarray(0, 1);
await new Promise((r) => setTimeout(r, 100));
yield value.bytes.subarray(1);
})(),
);
return { ...result, stream: slow };
};
await new Promise((resolve, reject) => {
const request = get(
`${f.url}/api/access/${value.item.accessId}/download`,
{ headers: { cookie: b.cookie, origin: 'http://hop.test' } },
(response) => {
assert.equal(response.statusCode, 200);
response.once('data', () => {
response.destroy();
request.destroy();
resolve();
});
},
);
request.once('error', reject);
});
if (!slow.closed) await once(slow, 'close');
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
f.service.storage.openDownload = openDownload;
assert.deepEqual(
(await f.call(b, `/access/${value.item.accessId}/download`)).rawPayload,
value.bytes,
);
await archived(f, value.row);
} finally {
await f.close();
}
});
test('journal recovers a restart between file rename and DB commit; unrelated files are not adopted', async () => {
const f = await fixture();
try {
const a = await f.create(),
b = await f.create();
const value = await file(f, a, b);
const key = `completed/${value.row.storageKey}`,
target = f.service.storage.layout.keyPath(key);
renameSync(f.service.storage.layout.path(value.row), target);
writeFileSync(
resolve(f.service.storage.layout.journal, value.row.id + '.json'),
JSON.stringify({
fileId: value.row.id,
checksum: value.row.checksum,
size: value.row.size,
from: value.row.storageKey,
to: key,
}),
);
f.service.storage.layout.prepare();
assert.equal(current(f, value.row).storageKey, key);
assert.deepEqual(
(await f.call(b, `/access/${value.item.accessId}/download`)).rawPayload,
value.bytes,
);
unlinkSync(target);
writeFileSync(
resolve(f.service.storage.originals, value.row.storageKey),
Buffer.alloc(value.row.size, 120),
);
const missing = await f.call(b, `/access/${value.item.accessId}/download`);
assert.equal(missing.statusCode, 404, 'no journal: do not adopt an unrelated same-size file');
} finally {
await f.close();
}
});
test('completed manual deletion and natural expiry clear references and remove bytes in either directory', async () => {
for (const manual of [true, false]) {
const f = await fixture();
try {
const a = await f.create(),
b = await f.create();
const value = await file(f, a, b);
await f.call(b, `/access/${value.item.accessId}/download`);
await archived(f, value.row);
const path = f.service.storage.layout.path(current(f, value.row));
if (manual) {
unlinkSync(path);
assert.equal((await f.call(b, `/access/${value.item.accessId}/download`)).statusCode, 404);
} else
f.service.db
.update(s.items)
.set({ expiresAt: Date.now() - 1 })
.where(eq(s.items.fileId, value.row.id))
.run();
await f.service.storage.cleanup();
assert.equal(existsSync(path), false);
assert.equal(current(f, value.row).status, 'failed');
} finally {
await f.close();
}
}
});