Separate completed transfers and retain original file extensions
This commit is contained in:
@@ -623,7 +623,7 @@ export async function createApp(cfg: Config, options: { mailer?: Mailer } = {})
|
||||
app.get('/api/access/:id/download', async (req, reply) => {
|
||||
const accessId = id(req),
|
||||
g = accessSubject(req, accessId);
|
||||
const { file, stream } = await service.storage.openDownload(g, accessId);
|
||||
const { file, stream, access } = await service.storage.openDownload(g, accessId);
|
||||
const fallback =
|
||||
file.name.replace(/[^a-zA-Z0-9._ -]/g, '_').replace(/^[. ]+/, '') || 'download';
|
||||
const encoded = encodeURIComponent(file.name).replace(
|
||||
@@ -643,6 +643,15 @@ export async function createApp(cfg: Config, options: { mailer?: Mailer } = {})
|
||||
stream.destroy();
|
||||
throw error;
|
||||
}
|
||||
reply.raw.once('finish', () => {
|
||||
if (reply.statusCode === 200 && stream.readableEnded && access.role === 'recipient') {
|
||||
try {
|
||||
service.storage.layout.downloaded(file.id);
|
||||
} catch (err) {
|
||||
req.log.warn({ err }, 'Completed file relocation will be retried');
|
||||
}
|
||||
}
|
||||
});
|
||||
return reply.send(stream);
|
||||
});
|
||||
app.get('/api/access/:id/preview', async (req, reply) =>
|
||||
|
||||
@@ -1,19 +1,29 @@
|
||||
import { statSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import { and, eq, gt, inArray } from 'drizzle-orm';
|
||||
import * as s from './db/schema.js';
|
||||
import { fail } from './security.js';
|
||||
|
||||
export class FileAvailability {
|
||||
constructor(service, directory) {
|
||||
constructor(service, filePath) {
|
||||
this.service = service;
|
||||
this.directory = directory;
|
||||
this.filePath = filePath;
|
||||
this.pending = new Set();
|
||||
}
|
||||
check(file) {
|
||||
try {
|
||||
if (statSync(resolve(this.directory, file.storageKey)).isFile()) return true;
|
||||
if (statSync(this.filePath(file)).isFile()) return true;
|
||||
} catch (error) {
|
||||
if (['ENOENT', 'ENOTDIR'].includes(error.code)) {
|
||||
try {
|
||||
if (this.service.storage.layout.recover(file)) return this.check(file);
|
||||
} catch {
|
||||
fail(
|
||||
503,
|
||||
'STORAGE_UNAVAILABLE',
|
||||
'파일 저장소에 접근할 수 없습니다. 잠시 후 다시 시도해 주세요.',
|
||||
);
|
||||
}
|
||||
}
|
||||
if (!['ENOENT', 'ENOTDIR'].includes(error.code))
|
||||
fail(
|
||||
503,
|
||||
|
||||
213
apps/server/src/fileLayout.js
Normal file
213
apps/server/src/fileLayout.js
Normal file
@@ -0,0 +1,213 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
renameSync,
|
||||
statSync,
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
unlinkSync,
|
||||
} from 'node:fs';
|
||||
import { basename, extname, resolve } from 'node:path';
|
||||
import { and, eq, gt, inArray, isNotNull, sql } from 'drizzle-orm';
|
||||
import * as s from './db/schema.js';
|
||||
|
||||
export function fileExtension(name) {
|
||||
const extension = extname(name).toLowerCase();
|
||||
return /^\.[a-z0-9]{1,16}$/.test(extension) ? extension : '';
|
||||
}
|
||||
export function newStorageKey(name) {
|
||||
return randomUUID() + fileExtension(name);
|
||||
}
|
||||
|
||||
export class FileLayout {
|
||||
constructor(service, canMove) {
|
||||
this.service = service;
|
||||
this.canMove = canMove;
|
||||
this.originals = resolve(service.cfg.uploadDir, 'originals');
|
||||
this.completed = resolve(service.cfg.uploadDir, 'completed');
|
||||
this.journal = resolve(service.cfg.uploadDir, '.file-moves');
|
||||
this.pending = new Set();
|
||||
for (const directory of [this.originals, this.completed, this.journal])
|
||||
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
||||
}
|
||||
path(file) {
|
||||
const current = this.service.db
|
||||
.select({ key: s.files.storageKey })
|
||||
.from(s.files)
|
||||
.where(eq(s.files.id, file.id))
|
||||
.get();
|
||||
return this.keyPath(current?.key ?? file.storageKey);
|
||||
}
|
||||
keyPath(key) {
|
||||
const name = key.startsWith('completed/') ? key.slice('completed/'.length) : key;
|
||||
if (!name || basename(name) !== name || ['.', '..'].includes(name))
|
||||
throw Error('Invalid stored file key');
|
||||
return resolve(key.startsWith('completed/') ? this.completed : this.originals, name);
|
||||
}
|
||||
hasAccountReference(id) {
|
||||
const { db, cfg } = this.service;
|
||||
return Boolean(
|
||||
db
|
||||
.select({ id: s.access.id })
|
||||
.from(s.items)
|
||||
.innerJoin(s.access, eq(s.access.itemId, s.items.id))
|
||||
.where(
|
||||
and(
|
||||
eq(s.items.fileId, id),
|
||||
isNotNull(s.access.userId),
|
||||
inArray(s.access.role, ['self', 'recipient']),
|
||||
gt(s.items.expiresAt, Date.now()),
|
||||
sql`(${s.access.deletedAt} IS NULL OR ${s.access.deletedAt} > ${Date.now() - cfg.undoSeconds * 1000})`,
|
||||
),
|
||||
)
|
||||
.get(),
|
||||
);
|
||||
}
|
||||
recover(file) {
|
||||
const journalPath = resolve(this.journal, file.id + '.json');
|
||||
if (!existsSync(journalPath)) return false;
|
||||
const entry = JSON.parse(readFileSync(journalPath, 'utf8'));
|
||||
if (entry.fileId !== file.id || entry.checksum !== file.checksum || entry.size !== file.size)
|
||||
throw Error('Invalid file relocation journal');
|
||||
const source = this.keyPath(entry.from),
|
||||
target = this.keyPath(entry.to);
|
||||
if (![entry.from, entry.to].includes(file.storageKey))
|
||||
throw Error('File relocation key changed');
|
||||
if (existsSync(file.storageKey === entry.from ? source : target)) {
|
||||
unlinkSync(journalPath);
|
||||
return false;
|
||||
}
|
||||
const recoveredKey = file.storageKey === entry.from ? entry.to : entry.from;
|
||||
const recoveredPath = this.keyPath(recoveredKey);
|
||||
if (existsSync(recoveredPath)) {
|
||||
const metadata = statSync(recoveredPath);
|
||||
if (!metadata.isFile() || metadata.size !== file.size)
|
||||
throw Error('Incomplete file relocation');
|
||||
this.service.db
|
||||
.update(s.files)
|
||||
.set({ storageKey: recoveredKey })
|
||||
.where(and(eq(s.files.id, file.id), eq(s.files.storageKey, file.storageKey)))
|
||||
.run();
|
||||
unlinkSync(journalPath);
|
||||
return true;
|
||||
}
|
||||
unlinkSync(journalPath);
|
||||
return false;
|
||||
}
|
||||
move(file, key) {
|
||||
if (key === file.storageKey || !this.canMove(file.id)) return;
|
||||
const destination = key;
|
||||
if (existsSync(this.keyPath(destination)))
|
||||
throw Error('File relocation destination already exists');
|
||||
const source = this.keyPath(file.storageKey),
|
||||
target = this.keyPath(destination);
|
||||
let moved = false;
|
||||
const journalPath = resolve(this.journal, file.id + '.json');
|
||||
writeFileSync(
|
||||
journalPath,
|
||||
JSON.stringify({
|
||||
fileId: file.id,
|
||||
checksum: file.checksum,
|
||||
size: file.size,
|
||||
from: file.storageKey,
|
||||
to: destination,
|
||||
}),
|
||||
{ mode: 0o600, flag: 'wx', flush: true },
|
||||
);
|
||||
try {
|
||||
this.service.db.transaction(() => {
|
||||
renameSync(source, target);
|
||||
moved = true;
|
||||
const updated = this.service.db
|
||||
.update(s.files)
|
||||
.set({ storageKey: destination })
|
||||
.where(
|
||||
and(
|
||||
eq(s.files.id, file.id),
|
||||
eq(s.files.storageKey, file.storageKey),
|
||||
inArray(s.files.status, ['stored', 'cleaning']),
|
||||
),
|
||||
)
|
||||
.run();
|
||||
if (!updated.changes) throw Error('Stored file changed during relocation');
|
||||
});
|
||||
} catch (error) {
|
||||
if (moved && !existsSync(source)) {
|
||||
try {
|
||||
renameSync(target, source);
|
||||
} catch {}
|
||||
}
|
||||
if (existsSync(source)) {
|
||||
try {
|
||||
unlinkSync(journalPath);
|
||||
} catch {}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
// A journal cleanup failure must never roll back a committed database/file move.
|
||||
try {
|
||||
unlinkSync(journalPath);
|
||||
} catch {}
|
||||
}
|
||||
prepare() {
|
||||
for (const file of this.service.db
|
||||
.select()
|
||||
.from(s.files)
|
||||
.where(inArray(s.files.status, ['stored', 'cleaning']))
|
||||
.all()) {
|
||||
try {
|
||||
this.recover(file);
|
||||
const current = this.service.db.select().from(s.files).where(eq(s.files.id, file.id)).get();
|
||||
const name = basename(current.storageKey),
|
||||
extension = fileExtension(current.name);
|
||||
if (extension && !extname(name) && existsSync(this.keyPath(current.storageKey)))
|
||||
this.move(
|
||||
current,
|
||||
(current.storageKey.startsWith('completed/') ? 'completed/' : '') + name + extension,
|
||||
);
|
||||
} catch {
|
||||
// Naming is cosmetic: keep the old key and let availability checks report storage errors.
|
||||
}
|
||||
}
|
||||
}
|
||||
downloaded(id) {
|
||||
this.pending.add(id);
|
||||
this.flush(id);
|
||||
}
|
||||
flush(id) {
|
||||
if (!this.pending.has(id) || !this.canMove(id)) return;
|
||||
const file = this.service.db.select().from(s.files).where(eq(s.files.id, id)).get();
|
||||
if (!file || file.status !== 'stored') {
|
||||
this.pending.delete(id);
|
||||
return;
|
||||
}
|
||||
const directory = this.hasAccountReference(id) ? '' : 'completed/';
|
||||
this.move(file, directory + basename(file.storageKey));
|
||||
this.pending.delete(id);
|
||||
}
|
||||
reconcile() {
|
||||
for (const file of this.service.db
|
||||
.selectDistinct({ id: s.files.id })
|
||||
.from(s.files)
|
||||
.innerJoin(s.items, eq(s.items.fileId, s.files.id))
|
||||
.innerJoin(s.access, eq(s.access.itemId, s.items.id))
|
||||
.where(
|
||||
and(
|
||||
eq(s.files.status, 'stored'),
|
||||
sql`${s.files.storageKey} LIKE 'completed/%'`,
|
||||
isNotNull(s.access.userId),
|
||||
inArray(s.access.role, ['self', 'recipient']),
|
||||
gt(s.items.expiresAt, Date.now()),
|
||||
sql`(${s.access.deletedAt} IS NULL OR ${s.access.deletedAt} > ${Date.now() - this.service.cfg.undoSeconds * 1000})`,
|
||||
),
|
||||
)
|
||||
.all())
|
||||
this.pending.add(file.id);
|
||||
for (const id of this.pending) {
|
||||
try {
|
||||
this.flush(id);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import type { Guest, Service } from './service.js';
|
||||
import * as s from './db/schema.js';
|
||||
import { ApiError, fail } from './security.js';
|
||||
import { FileAvailability } from './fileAvailability.js';
|
||||
import { FileLayout, newStorageKey } from './fileLayout.js';
|
||||
export type StorageSubject = { id: string; kind?: 'user'; sessionId?: string };
|
||||
export class Storage {
|
||||
private active = new Map<
|
||||
@@ -20,6 +21,8 @@ export class Storage {
|
||||
>();
|
||||
private cleaning = false;
|
||||
private cleanupPromise: Promise<void> | undefined;
|
||||
private readers = new Map<string, number>();
|
||||
readonly layout: FileLayout;
|
||||
readonly temp: string;
|
||||
readonly originals: string;
|
||||
readonly availability: FileAvailability;
|
||||
@@ -28,7 +31,11 @@ export class Storage {
|
||||
this.originals = resolve(service.cfg.uploadDir, 'originals');
|
||||
mkdirSync(this.temp, { recursive: true, mode: 0o700 });
|
||||
mkdirSync(this.originals, { recursive: true, mode: 0o700 });
|
||||
this.availability = new FileAvailability(service, this.originals);
|
||||
this.layout = new FileLayout(service, (id: string) => !this.readers.has(id));
|
||||
this.layout.prepare();
|
||||
this.availability = new FileAvailability(service, (file: typeof s.files.$inferSelect) =>
|
||||
this.layout.path(file),
|
||||
);
|
||||
}
|
||||
private request(g: StorageSubject, id: string) {
|
||||
return g.kind === 'user'
|
||||
@@ -68,7 +75,7 @@ export class Storage {
|
||||
name: file.name,
|
||||
size: file.size,
|
||||
type: file.type,
|
||||
storageKey: randomUUID(),
|
||||
storageKey: newStorageKey(file.name),
|
||||
status: 'pending',
|
||||
updatedAt: Date.now(),
|
||||
})
|
||||
@@ -434,14 +441,29 @@ export class Storage {
|
||||
if (row.item.kind !== 'file' || !row.item.fileId)
|
||||
fail(404, 'FILE_NOT_FOUND', '파일을 찾을 수 없습니다.');
|
||||
const file = this.availability.require(row.item.fileId!);
|
||||
return { file, path: resolve(this.originals, file.storageKey) };
|
||||
return { file, path: this.layout.path(file), access: row.access };
|
||||
}
|
||||
private beginRead(id: string) {
|
||||
this.readers.set(id, (this.readers.get(id) ?? 0) + 1);
|
||||
}
|
||||
private endRead(id: string) {
|
||||
const remaining = (this.readers.get(id) ?? 1) - 1;
|
||||
if (remaining) this.readers.set(id, remaining);
|
||||
else this.readers.delete(id);
|
||||
try {
|
||||
this.layout.flush(id);
|
||||
} catch {}
|
||||
}
|
||||
async openDownload(g: StorageSubject, accessId: string) {
|
||||
const { file, path } = this.download(g, accessId);
|
||||
const { file, path, access } = this.download(g, accessId);
|
||||
this.beginRead(file.id);
|
||||
try {
|
||||
const handle = await open(path, 'r');
|
||||
return { file, stream: handle.createReadStream() };
|
||||
const stream = handle.createReadStream();
|
||||
stream.once('close', () => this.endRead(file.id));
|
||||
return { file, stream, access };
|
||||
} catch (error) {
|
||||
this.endRead(file.id);
|
||||
if (
|
||||
error &&
|
||||
typeof error === 'object' &&
|
||||
@@ -463,7 +485,8 @@ export class Storage {
|
||||
}
|
||||
}
|
||||
async preview(g: StorageSubject, id: string) {
|
||||
const { path } = this.download(g, id);
|
||||
const { file, path } = this.download(g, id);
|
||||
this.beginRead(file.id);
|
||||
try {
|
||||
const handle = await open(path, 'r');
|
||||
const magic = Buffer.alloc(16);
|
||||
@@ -491,6 +514,8 @@ export class Storage {
|
||||
.toBuffer();
|
||||
} catch {
|
||||
fail(415, 'PREVIEW_UNSUPPORTED', '이 파일은 이미지 미리보기를 지원하지 않습니다.');
|
||||
} finally {
|
||||
this.endRead(file.id);
|
||||
}
|
||||
}
|
||||
deleteAccess(g: StorageSubject, ids: string[]) {
|
||||
@@ -583,6 +608,7 @@ export class Storage {
|
||||
this.service.resumable?.cleanup();
|
||||
const { db, cfg } = this.service,
|
||||
now = Date.now();
|
||||
this.layout.reconcile();
|
||||
const expired = db
|
||||
.select()
|
||||
.from(s.reservations)
|
||||
@@ -622,7 +648,7 @@ export class Storage {
|
||||
if (refs.length) continue;
|
||||
db.update(s.files).set({ status: 'cleaning' }).where(eq(s.files.id, file.id)).run();
|
||||
try {
|
||||
await unlink(resolve(this.originals, file.storageKey));
|
||||
await unlink(this.layout.path(file));
|
||||
} catch (error) {
|
||||
if (!error || typeof error !== 'object' || !('code' in error) || error.code !== 'ENOENT')
|
||||
continue;
|
||||
@@ -640,11 +666,38 @@ export class Storage {
|
||||
.all()
|
||||
.map((f) => f.storageKey),
|
||||
);
|
||||
for (const name of await readdir(this.originals)) {
|
||||
if (validKeys.has(name)) continue;
|
||||
const path = resolve(this.originals, name),
|
||||
metadata = await stat(path);
|
||||
if (metadata.mtimeMs < now - cfg.orphanHours * 3600000) await unlink(path).catch(() => {});
|
||||
for (const [directory, prefix] of [
|
||||
[this.originals, ''],
|
||||
[this.layout.completed, 'completed/'],
|
||||
]) {
|
||||
for (const name of await readdir(directory)) {
|
||||
const key = prefix + name;
|
||||
if (validKeys.has(key)) continue;
|
||||
const path = resolve(directory, name);
|
||||
try {
|
||||
const metadata = await stat(path);
|
||||
if (!metadata.isFile() || metadata.mtimeMs >= now - cfg.orphanHours * 3600000) continue;
|
||||
if (
|
||||
db
|
||||
.select()
|
||||
.from(s.files)
|
||||
.where(
|
||||
and(eq(s.files.storageKey, key), inArray(s.files.status, ['stored', 'cleaning'])),
|
||||
)
|
||||
.get()
|
||||
)
|
||||
continue;
|
||||
await unlink(path).catch(() => {});
|
||||
} catch (error) {
|
||||
if (
|
||||
!error ||
|
||||
typeof error !== 'object' ||
|
||||
!('code' in error) ||
|
||||
!['ENOENT', 'ENOTDIR'].includes(String(error.code))
|
||||
)
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const name of await readdir(this.temp)) {
|
||||
if (this.active.has(name)) continue;
|
||||
|
||||
393
apps/server/test/file-layout.test.js
Normal file
393
apps/server/test/file-layout.test.js
Normal file
@@ -0,0 +1,393 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { once } from 'node:events';
|
||||
import { get } from 'node:http';
|
||||
import { Readable } from 'node:stream';
|
||||
import { existsSync, readFileSync, renameSync, writeFileSync, unlinkSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
import { and, eq, isNotNull } from 'drizzle-orm';
|
||||
import { fixture } from './helpers.js';
|
||||
import * as s from '../src/db/schema.js';
|
||||
|
||||
async function login(f, c) {
|
||||
const id = randomUUID(),
|
||||
email = `${id}@hop.test`;
|
||||
f.service.db
|
||||
.insert(s.users)
|
||||
.values({
|
||||
id,
|
||||
email,
|
||||
displayName: email,
|
||||
passwordHash: await f.service.auth.password('12345678'),
|
||||
status: 'active',
|
||||
verifiedAt: Date.now(),
|
||||
createdAt: Date.now(),
|
||||
})
|
||||
.run();
|
||||
const r = await f.call(c, '/auth/login', 'POST', { email, password: '12345678' });
|
||||
assert.equal(r.statusCode, 200);
|
||||
c.cookie += '; ' + String(r.headers['set-cookie']).split(';')[0];
|
||||
return id;
|
||||
}
|
||||
async function file(f, a, b, name = '사진.JPG') {
|
||||
const bytes = Buffer.from('unchanged file bytes');
|
||||
await f.space(a, b);
|
||||
const r = await f.call(
|
||||
a,
|
||||
'/transfer-requests',
|
||||
'POST',
|
||||
{
|
||||
targetDeviceId: b.deviceId,
|
||||
kind: 'file',
|
||||
count: 1,
|
||||
byteSize: bytes.length,
|
||||
files: [{ name, size: bytes.length, type: 'application/octet-stream' }],
|
||||
},
|
||||
randomUUID(),
|
||||
);
|
||||
assert.equal(r.statusCode, 200, r.body);
|
||||
const id = r.json().requestId;
|
||||
assert.equal(
|
||||
(await f.call(b, `/transfer-requests/${id}/accept`, 'POST', {}, randomUUID())).statusCode,
|
||||
200,
|
||||
);
|
||||
const boundary = 'layout-upload',
|
||||
body = Buffer.concat([
|
||||
Buffer.from(
|
||||
`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${name}"\r\nContent-Type: application/octet-stream\r\n\r\n`,
|
||||
),
|
||||
bytes,
|
||||
Buffer.from(`\r\n--${boundary}--\r\n`),
|
||||
]);
|
||||
const upload = await f.call(
|
||||
a,
|
||||
`/uploads?requestId=${id}&fileIndex=0`,
|
||||
'POST',
|
||||
body,
|
||||
randomUUID(),
|
||||
{ 'content-type': `multipart/form-data; boundary=${boundary}` },
|
||||
);
|
||||
assert.equal(upload.statusCode, 200, upload.body);
|
||||
const row = f.service.storage.fileRows(id)[0];
|
||||
const item = (await f.call(b, '/device/items')).json().items.find((i) => i.file?.id === row.id);
|
||||
return { row, item, bytes, id };
|
||||
}
|
||||
function current(f, row) {
|
||||
return f.service.db.select().from(s.files).where(eq(s.files.id, row.id)).get();
|
||||
}
|
||||
async function archived(f, row) {
|
||||
// The HTTP response can finish before the file descriptor's asynchronous close.
|
||||
for (let i = 0; i < 100 && !current(f, row).storageKey.startsWith('completed/'); i++)
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
assert.equal(current(f, row).storageKey, `completed/${row.storageKey}`);
|
||||
}
|
||||
|
||||
test('logged-in sender to anonymous recipient: extension kept, completed after download, seven-day expiry and re-download unchanged', async () => {
|
||||
const f = await fixture();
|
||||
try {
|
||||
const a = await f.create(),
|
||||
b = await f.create();
|
||||
await login(f, a);
|
||||
const value = await file(f, a, b),
|
||||
row = value.row;
|
||||
assert.match(row.storageKey, /^[a-f0-9-]{36}\.jpg$/);
|
||||
assert.equal((await f.call(a, '/vault')).json().items.length, 0);
|
||||
const expiry = f.service.db
|
||||
.select()
|
||||
.from(s.items)
|
||||
.where(eq(s.items.fileId, row.id))
|
||||
.get().expiresAt;
|
||||
assert.equal(existsSync(resolve(f.service.storage.originals, row.storageKey)), true);
|
||||
const downloaded = await f.call(b, `/access/${value.item.accessId}/download`);
|
||||
assert.equal(downloaded.statusCode, 200);
|
||||
assert.deepEqual(downloaded.rawPayload, value.bytes);
|
||||
await archived(f, row);
|
||||
const stored = current(f, row);
|
||||
assert.equal(stored.storageKey, `completed/${row.storageKey}`);
|
||||
assert.equal(existsSync(resolve(f.service.storage.originals, row.storageKey)), false);
|
||||
assert.deepEqual(readFileSync(f.service.storage.layout.path(stored)), value.bytes);
|
||||
assert.equal(stored.checksum, row.checksum);
|
||||
assert.equal(stored.updatedAt, row.updatedAt);
|
||||
assert.equal(
|
||||
f.service.db.select().from(s.items).where(eq(s.items.fileId, row.id)).get().expiresAt,
|
||||
expiry,
|
||||
);
|
||||
assert.deepEqual(
|
||||
(await f.call(b, `/access/${value.item.accessId}/download`)).rawPayload,
|
||||
value.bytes,
|
||||
);
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
test('anonymous sender to signed-in recipient: received-account reference keeps the file in originals', async () => {
|
||||
const f = await fixture();
|
||||
try {
|
||||
const a = await f.create(),
|
||||
b = await f.create();
|
||||
await login(f, b);
|
||||
const value = await file(f, a, b, 'received.pdf');
|
||||
const downloaded = await f.call(b, `/access/${value.item.accessId}/download`);
|
||||
assert.equal(downloaded.statusCode, 200);
|
||||
await new Promise((r) => setImmediate(r));
|
||||
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
|
||||
assert(existsSync(f.service.storage.layout.path(value.row)));
|
||||
assert.equal((await f.call(b, '/inbox')).json().items.length, 1);
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
test('a sender read or unauthorized read does not classify a transfer as recipient-completed', async () => {
|
||||
const f = await fixture();
|
||||
try {
|
||||
const a = await f.create(),
|
||||
b = await f.create(),
|
||||
c = await f.create();
|
||||
const value = await file(f, a, b);
|
||||
assert.equal((await f.call(c, `/access/${value.item.accessId}/download`)).statusCode, 404);
|
||||
const sent = (await f.call(a, '/device/items')).json().items[0];
|
||||
assert.equal((await f.call(a, `/access/${sent.accessId}/download`)).statusCode, 200);
|
||||
await new Promise((r) => setImmediate(r));
|
||||
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
test('relocation waits for all active readers and preserves concurrent download bytes', async () => {
|
||||
const f = await fixture();
|
||||
try {
|
||||
const a = await f.create(),
|
||||
b = await f.create();
|
||||
const value = await file(f, a, b);
|
||||
const guest = f.service.db
|
||||
.select()
|
||||
.from(s.guests)
|
||||
.where(eq(s.guests.id, f.service.deviceGuest(b.deviceId).guestId))
|
||||
.get();
|
||||
const first = await f.service.storage.openDownload(guest, value.item.accessId),
|
||||
second = await f.service.storage.openDownload(guest, value.item.accessId);
|
||||
f.service.storage.layout.downloaded(value.row.id);
|
||||
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
|
||||
const firstClosed = once(first.stream, 'close');
|
||||
first.stream.resume();
|
||||
await firstClosed;
|
||||
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
|
||||
const chunks = [];
|
||||
second.stream.on('data', (chunk) => chunks.push(chunk));
|
||||
await once(second.stream, 'close');
|
||||
assert.deepEqual(Buffer.concat(chunks), value.bytes);
|
||||
assert.equal(current(f, value.row).storageKey, `completed/${value.row.storageKey}`);
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
test('legacy extension-less files migrate without changing IDs, checksums, content or expiry', async () => {
|
||||
const f = await fixture();
|
||||
try {
|
||||
const a = await f.create(),
|
||||
b = await f.create();
|
||||
const value = await file(f, a, b, 'old.png');
|
||||
const legacy = randomUUID();
|
||||
renameSync(
|
||||
f.service.storage.layout.path(value.row),
|
||||
resolve(f.service.storage.originals, legacy),
|
||||
);
|
||||
f.service.db
|
||||
.update(s.files)
|
||||
.set({ storageKey: legacy })
|
||||
.where(eq(s.files.id, value.row.id))
|
||||
.run();
|
||||
f.service.storage.layout.prepare();
|
||||
const stored = current(f, value.row);
|
||||
assert.equal(stored.storageKey, legacy + '.png');
|
||||
assert.equal(stored.id, value.row.id);
|
||||
assert.deepEqual(readFileSync(f.service.storage.layout.path(stored)), value.bytes);
|
||||
assert.equal(stored.checksum, value.row.checksum);
|
||||
const before = stored.storageKey;
|
||||
f.service.storage.layout.prepare();
|
||||
assert.equal(current(f, value.row).storageKey, before);
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
test('database relocation failure rolls back the filename and preserves the original bytes', async () => {
|
||||
const f = await fixture();
|
||||
try {
|
||||
const a = await f.create(),
|
||||
b = await f.create();
|
||||
const value = await file(f, a, b);
|
||||
f.sqlite.exec(
|
||||
"CREATE TRIGGER reject_move BEFORE UPDATE OF storageKey ON files BEGIN SELECT RAISE(ABORT, 'test relocation failure'); END",
|
||||
);
|
||||
assert.throws(() =>
|
||||
f.service.storage.layout.move(value.row, `completed/${value.row.storageKey}`),
|
||||
);
|
||||
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
|
||||
assert.deepEqual(readFileSync(f.service.storage.layout.path(value.row)), value.bytes);
|
||||
assert.equal(
|
||||
existsSync(f.service.storage.layout.keyPath(`completed/${value.row.storageKey}`)),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
existsSync(resolve(f.service.storage.layout.journal, value.row.id + '.json')),
|
||||
false,
|
||||
);
|
||||
f.sqlite.exec('DROP TRIGGER reject_move');
|
||||
f.service.storage.layout.downloaded(value.row.id);
|
||||
assert.equal(current(f, value.row).storageKey, `completed/${value.row.storageKey}`);
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
test('account deletion undo remains protected; restoring an account reference returns an archive to originals', async () => {
|
||||
const f = await fixture();
|
||||
try {
|
||||
const a = await f.create(),
|
||||
b = await f.create();
|
||||
await login(f, b);
|
||||
const value = await file(f, a, b);
|
||||
const reference = f.service.db
|
||||
.select()
|
||||
.from(s.access)
|
||||
.where(and(eq(s.access.itemId, value.item.id), isNotNull(s.access.userId)))
|
||||
.get();
|
||||
assert(reference);
|
||||
f.service.db
|
||||
.update(s.access)
|
||||
.set({ deletedAt: Date.now() })
|
||||
.where(eq(s.access.id, reference.id))
|
||||
.run();
|
||||
f.service.storage.layout.downloaded(value.row.id);
|
||||
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
|
||||
f.service.db
|
||||
.update(s.access)
|
||||
.set({ deletedAt: Date.now() - f.cfg.undoSeconds * 1000 - 1 })
|
||||
.where(eq(s.access.id, reference.id))
|
||||
.run();
|
||||
f.service.storage.layout.downloaded(value.row.id);
|
||||
assert.equal(current(f, value.row).storageKey, `completed/${value.row.storageKey}`);
|
||||
f.service.db
|
||||
.update(s.access)
|
||||
.set({ deletedAt: null })
|
||||
.where(eq(s.access.id, reference.id))
|
||||
.run();
|
||||
f.service.storage.layout.reconcile();
|
||||
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
|
||||
assert.deepEqual(readFileSync(f.service.storage.layout.path(value.row)), value.bytes);
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
test('aborting an HTTP download before all bytes arrive leaves the file available in originals', async () => {
|
||||
const f = await fixture();
|
||||
try {
|
||||
const a = await f.create(),
|
||||
b = await f.create();
|
||||
const value = await file(f, a, b);
|
||||
const openDownload = f.service.storage.openDownload.bind(f.service.storage);
|
||||
let slow;
|
||||
f.service.storage.openDownload = async (...args) => {
|
||||
const result = await openDownload(...args);
|
||||
const closed = once(result.stream, 'close');
|
||||
result.stream.destroy();
|
||||
await closed;
|
||||
slow = Readable.from(
|
||||
(async function* () {
|
||||
yield value.bytes.subarray(0, 1);
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
yield value.bytes.subarray(1);
|
||||
})(),
|
||||
);
|
||||
return { ...result, stream: slow };
|
||||
};
|
||||
await new Promise((resolve, reject) => {
|
||||
const request = get(
|
||||
`${f.url}/api/access/${value.item.accessId}/download`,
|
||||
{ headers: { cookie: b.cookie, origin: 'http://hop.test' } },
|
||||
(response) => {
|
||||
assert.equal(response.statusCode, 200);
|
||||
response.once('data', () => {
|
||||
response.destroy();
|
||||
request.destroy();
|
||||
resolve();
|
||||
});
|
||||
},
|
||||
);
|
||||
request.once('error', reject);
|
||||
});
|
||||
if (!slow.closed) await once(slow, 'close');
|
||||
assert.equal(current(f, value.row).storageKey, value.row.storageKey);
|
||||
f.service.storage.openDownload = openDownload;
|
||||
assert.deepEqual(
|
||||
(await f.call(b, `/access/${value.item.accessId}/download`)).rawPayload,
|
||||
value.bytes,
|
||||
);
|
||||
await archived(f, value.row);
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
test('journal recovers a restart between file rename and DB commit; unrelated files are not adopted', async () => {
|
||||
const f = await fixture();
|
||||
try {
|
||||
const a = await f.create(),
|
||||
b = await f.create();
|
||||
const value = await file(f, a, b);
|
||||
const key = `completed/${value.row.storageKey}`,
|
||||
target = f.service.storage.layout.keyPath(key);
|
||||
renameSync(f.service.storage.layout.path(value.row), target);
|
||||
writeFileSync(
|
||||
resolve(f.service.storage.layout.journal, value.row.id + '.json'),
|
||||
JSON.stringify({
|
||||
fileId: value.row.id,
|
||||
checksum: value.row.checksum,
|
||||
size: value.row.size,
|
||||
from: value.row.storageKey,
|
||||
to: key,
|
||||
}),
|
||||
);
|
||||
f.service.storage.layout.prepare();
|
||||
assert.equal(current(f, value.row).storageKey, key);
|
||||
assert.deepEqual(
|
||||
(await f.call(b, `/access/${value.item.accessId}/download`)).rawPayload,
|
||||
value.bytes,
|
||||
);
|
||||
unlinkSync(target);
|
||||
writeFileSync(
|
||||
resolve(f.service.storage.originals, value.row.storageKey),
|
||||
Buffer.alloc(value.row.size, 120),
|
||||
);
|
||||
const missing = await f.call(b, `/access/${value.item.accessId}/download`);
|
||||
assert.equal(missing.statusCode, 404, 'no journal: do not adopt an unrelated same-size file');
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
});
|
||||
test('completed manual deletion and natural expiry clear references and remove bytes in either directory', async () => {
|
||||
for (const manual of [true, false]) {
|
||||
const f = await fixture();
|
||||
try {
|
||||
const a = await f.create(),
|
||||
b = await f.create();
|
||||
const value = await file(f, a, b);
|
||||
await f.call(b, `/access/${value.item.accessId}/download`);
|
||||
await archived(f, value.row);
|
||||
const path = f.service.storage.layout.path(current(f, value.row));
|
||||
if (manual) {
|
||||
unlinkSync(path);
|
||||
assert.equal((await f.call(b, `/access/${value.item.accessId}/download`)).statusCode, 404);
|
||||
} else
|
||||
f.service.db
|
||||
.update(s.items)
|
||||
.set({ expiresAt: Date.now() - 1 })
|
||||
.where(eq(s.items.fileId, value.row.id))
|
||||
.run();
|
||||
await f.service.storage.cleanup();
|
||||
assert.equal(existsSync(path), false);
|
||||
assert.equal(current(f, value.row).status, 'failed');
|
||||
} finally {
|
||||
await f.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user