diff --git a/src/commands/download.tsx b/src/commands/download.tsx new file mode 100644 index 0000000..ba9a9a6 --- /dev/null +++ b/src/commands/download.tsx @@ -0,0 +1,196 @@ +import React, { useEffect, useState } from "react"; +import { Box, Text, useApp } from "ink"; +import { flag, flagAs, intFlag, requireArg, type Flags } from "../lib/args"; +import { truncate } from "../lib/format"; +import { KeyHints, Panel, ScreenFrame } from "../components/ScreenChrome"; +import { Spinner } from "../components/Spinner"; +import { cancellationSignal } from "../lib/network"; +import { + executeDownload, + downloadExitCode, + type DownloadEvent, + type DownloadSummary, + type ImageSize, +} from "../lib/download"; + +export interface DownloadCommandOptions { + slug: string; + directory?: string; + size: ImageSize; + concurrency: number; + includeText: boolean; + type?: string; + overwrite: boolean; +} + +export function parseDownloadOptions( + args: string[], + flags: Flags, +): DownloadCommandOptions { + return { + slug: requireArg(args, 0, "slug"), + directory: flag(flags, "dir"), + size: flagAs(flags, "size") ?? "original", + concurrency: intFlag(flags, "concurrency") ?? 4, + includeText: flags["include-text"] !== undefined, + type: flag(flags, "type"), + overwrite: flags["overwrite"] !== undefined, + }; +} + +function progressBar(completed: number, total: number, width = 28): string { + if (total <= 0) return `[${"░".repeat(width)}]`; + const ratio = Math.max(0, Math.min(1, completed / total)); + const filled = Math.round(ratio * width); + return `[${"█".repeat(filled)}${"░".repeat(width - filled)}]`; +} + +type Phase = "listing" | "downloading" | "done" | "error"; + +export function DownloadCommand(options: DownloadCommandOptions) { + const { exit } = useApp(); + const [phase, setPhase] = useState("listing"); + const [error, setError] = useState(null); + const [summary, setSummary] = useState(null); + const [listed, setListed] = useState(0); + const [progress, setProgress] = useState({ + completed: 0, + total: 0, + downloaded: 0, + skipped: 0, + failed: 0, + }); + const [recent, setRecent] = useState([]); + + useEffect(() => { + let cancelled = false; + + const onEvent = (event: DownloadEvent) => { + if (cancelled) return; + switch (event.type) { + case "list_progress": + setListed(event.fetched); + break; + case "list_completed": + setProgress((prev) => ({ ...prev, total: event.downloadable })); + setPhase("downloading"); + break; + case "file_progress": + setProgress({ + completed: event.completed, + total: event.total, + downloaded: event.downloaded, + skipped: event.skipped, + failed: event.failed, + }); + setRecent((prev) => [event.file, ...prev].slice(0, 8)); + break; + } + }; + + const run = async () => { + try { + const result = await executeDownload({ + ...options, + onEvent, + signal: cancellationSignal(), + }); + if (cancelled) return; + setSummary(result); + if (downloadExitCode(result) !== 0) process.exitCode = 1; + setPhase("done"); + } catch (err: unknown) { + if (cancelled) return; + setError(err instanceof Error ? err.message : String(err)); + process.exitCode = 1; + setPhase("error"); + } + }; + + void run(); + return () => { + cancelled = true; + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + useEffect(() => { + if (phase === "done" || phase === "error") exit(); + }, [phase, exit]); + + if (phase === "listing") { + return ; + } + + if (phase === "error") { + return ✕ {error ?? "Download failed"}; + } + + if (phase === "downloading") { + return ( + + + + concurrency {options.concurrency} · size {options.size} + + + + + {progress.completed}/{progress.total}{" "} + {progressBar(progress.completed, progress.total)} + + + downloaded: {progress.downloaded} · skipped: {progress.skipped}{" "} + · failed: {progress.failed} + + + + {recent.length > 0 ? ( + + + {recent.map((file, index) => ( + · {truncate(file, 84)} + ))} + + + ) : null} + + + + ); + } + + if (!summary) return null; + + return ( + + + + {downloadExitCode(summary) === 0 ? "✓" : "!"} Download finished + + + listed {summary.listed} · downloaded {summary.downloaded} · skipped{" "} + {summary.skipped} · failed {summary.failed} + + → {summary.directory} + {summary.failures.slice(0, 3).map((failure, index) => ( + + fail: {truncate(failure.file, 64)} · {failure.error} + + ))} + + + ); +} + +export async function runDownloadJsonStream( + options: DownloadCommandOptions, + write: (event: DownloadEvent) => void, +): Promise { + const summary = await executeDownload({ + ...options, + onEvent: write, + signal: cancellationSignal(), + }); + return downloadExitCode(summary); +} diff --git a/src/lib/download.test.ts b/src/lib/download.test.ts new file mode 100644 index 0000000..eaabc80 --- /dev/null +++ b/src/lib/download.test.ts @@ -0,0 +1,387 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import type { Block, Connectable, PaginationMeta } from "../api/types"; +import { + buildManifest, + executeDownload, + downloadExitCode, + resolveTarget, + type DownloadAdapter, + type DownloadEvent, + type DownloadOptions, +} from "./download"; + +// ── Minimal block factories ──────────────────────────────────────────────── +// resolveTarget/buildManifest only touch a handful of fields, so we cast small +// partials rather than constructing the full discriminated union. + +function imageBlock( + id: number, + overrides: Record = {}, +): Block { + return { + id, + base_type: "Block", + type: "Image", + created_at: "2023-01-01T00:00:00Z", + image: { + src: `https://cdn.example/${id}/original.jpg`, + large: { src: `https://cdn.example/${id}/large.jpg`, src_2x: "" }, + filename: `photo-${id}.jpg`, + content_type: "image/jpeg", + }, + ...overrides, + } as unknown as Block; +} + +function attachmentBlock(id: number): Block { + return { + id, + base_type: "Block", + type: "Attachment", + created_at: "2023-01-01T00:00:00Z", + attachment: { + url: `https://attachments.example/${id}/doc.pdf`, + filename: `doc-${id}.pdf`, + content_type: "application/pdf", + }, + } as unknown as Block; +} + +function textBlock(id: number): Block { + return { + id, + base_type: "Block", + type: "Text", + created_at: "2023-01-01T00:00:00Z", + content: { markdown: `# note ${id}`, html: "", plain: `note ${id}` }, + } as unknown as Block; +} + +function linkBlock(id: number, url: string): Block { + return { + id, + base_type: "Block", + type: "Link", + title: `link ${id}`, + created_at: "2023-01-01T00:00:00Z", + source: { url }, + } as unknown as Block; +} + +const baseOptions: DownloadOptions = { + slug: "test-channel", + size: "original", + concurrency: 2, + includeText: false, + overwrite: false, +}; + +function meta(overrides: Partial = {}): PaginationMeta { + return { + current_page: 1, + per_page: 100, + total_pages: 1, + total_count: 0, + has_more_pages: false, + ...overrides, + }; +} + +/** Adapter whose assets are in-memory buffers; records which URLs were fetched. */ +function fakeAdapter( + pages: Array<{ data: Connectable[]; meta: PaginationMeta }>, + opts: { + onOpen?: (url: string) => void; + failUrls?: Map; // url -> times to throw before succeeding + } = {}, +): DownloadAdapter { + let pageCursor = 0; + return { + async listContents() { + return pages[pageCursor++] ?? { data: [], meta: meta() }; + }, + async openAsset(url) { + opts.onOpen?.(url); + const remaining = opts.failUrls?.get(url) ?? 0; + if (remaining > 0) { + opts.failUrls!.set(url, remaining - 1); + throw new Error("fetch failed"); // transient + } + return Readable.from([Buffer.from(`bytes:${url}`)]); + }, + async sleep() {}, + }; +} + +// ── resolveTarget ─────────────────────────────────────────────────────────── + +test("resolveTarget maps an image to its original URL with a position prefix", () => { + const target = resolveTarget(imageBlock(7), 0, baseOptions); + assert.deepEqual(target, { + kind: "fetch", + url: "https://cdn.example/7/original.jpg", + filename: "0001_photo-7.jpg", + }); +}); + +test("resolveTarget honors --size by selecting a resized version", () => { + const target = resolveTarget(imageBlock(7), 4, { + ...baseOptions, + size: "large", + }); + assert.equal(target?.kind, "fetch"); + assert.equal( + target && target.kind === "fetch" ? target.url : null, + "https://cdn.example/7/large.jpg", + ); + assert.equal(target?.filename, "0005_photo-7.jpg"); +}); + +test("resolveTarget falls back to id + content-type extension when filename is absent", () => { + const block = imageBlock(9, { + image: { + src: "https://cdn.example/9/original.jpg", + content_type: "image/png", + }, + }); + const target = resolveTarget(block, 0, baseOptions); + assert.equal(target?.filename, "0001_9.png"); +}); + +test("resolveTarget maps attachments to their download URL", () => { + const target = resolveTarget(attachmentBlock(3), 1, baseOptions); + assert.deepEqual(target, { + kind: "fetch", + url: "https://attachments.example/3/doc.pdf", + filename: "0002_doc-3.pdf", + }); +}); + +test("resolveTarget skips text blocks unless includeText is set", () => { + assert.equal(resolveTarget(textBlock(1), 0, baseOptions), null); + const target = resolveTarget(textBlock(1), 0, { + ...baseOptions, + includeText: true, + }); + assert.deepEqual(target, { + kind: "text", + content: "# note 1", + filename: "0001_1.md", + }); +}); + +test("resolveTarget skips link blocks (no hosted file)", () => { + assert.equal( + resolveTarget(linkBlock(1, "https://example.com"), 0, baseOptions), + null, + ); +}); + +// ── buildManifest ───────────────────────────────────────────────────────── + +test("buildManifest captures every block including link source URLs", () => { + const manifest = buildManifest([ + imageBlock(1), + linkBlock(2, "https://example.com/ref"), + ]); + assert.equal(manifest.length, 2); + assert.equal(manifest[0]?.type, "Image"); + assert.equal(manifest[0]?.filename, "photo-1.jpg"); + assert.equal(manifest[1]?.type, "Link"); + assert.equal(manifest[1]?.source_url, "https://example.com/ref"); +}); + +// ── executeDownload ───────────────────────────────────────────────────────── + +test("executeDownload downloads assets, writes a manifest, and emits completed", async () => { + const dir = await mkdtemp(join(tmpdir(), "arena-dl-")); + try { + const blocks = [ + imageBlock(1), + attachmentBlock(2), + linkBlock(3, "https://x.io"), + ]; + const adapter = fakeAdapter([ + { data: blocks, meta: meta({ total_count: 3 }) }, + ]); + const events: DownloadEvent[] = []; + + const summary = await executeDownload({ + ...baseOptions, + directory: dir, + adapter, + onEvent: (event) => { + events.push(event); + }, + }); + + assert.equal(summary.listed, 3); + assert.equal(summary.downloaded, 2); // image + attachment; link is skipped + assert.equal(summary.failed, 0); + assert.equal(downloadExitCode(summary), 0); + + const files = (await readdir(dir)).sort(); + assert.deepEqual(files, [ + "0001_photo-1.jpg", + "0002_doc-2.pdf", + "manifest.json", + ]); + + const manifest = JSON.parse( + await readFile(join(dir, "manifest.json"), "utf-8"), + ); + assert.equal(manifest.length, 3); // all blocks recorded, including the link + + assert.equal(events.at(-1)?.type, "completed"); + assert.ok(events.some((e) => e.type === "list_completed")); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("executeDownload paginates across pages", async () => { + const dir = await mkdtemp(join(tmpdir(), "arena-dl-")); + try { + const adapter = fakeAdapter([ + { + data: [imageBlock(1)], + meta: meta({ + total_count: 2, + total_pages: 2, + has_more_pages: true, + next_page: 2, + }), + }, + { + data: [imageBlock(2)], + meta: meta({ current_page: 2, total_count: 2, total_pages: 2 }), + }, + ]); + + const summary = await executeDownload({ + ...baseOptions, + directory: dir, + adapter, + }); + assert.equal(summary.listed, 2); + assert.equal(summary.downloaded, 2); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("executeDownload records failures and returns a non-zero exit code", async () => { + const dir = await mkdtemp(join(tmpdir(), "arena-dl-")); + try { + // Permanent failure (more than the retry budget) on one asset. + const failUrls = new Map([["https://cdn.example/2/original.jpg", 99]]); + const adapter = fakeAdapter( + [ + { + data: [imageBlock(1), imageBlock(2)], + meta: meta({ total_count: 2 }), + }, + ], + { failUrls }, + ); + + const summary = await executeDownload({ + ...baseOptions, + concurrency: 1, + directory: dir, + adapter, + }); + + assert.equal(summary.downloaded, 1); + assert.equal(summary.failed, 1); + assert.equal(summary.failures[0]?.file, "0002_photo-2.jpg"); + assert.equal(downloadExitCode(summary), 1); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("executeDownload retries transient asset failures", async () => { + const dir = await mkdtemp(join(tmpdir(), "arena-dl-")); + try { + const failUrls = new Map([["https://cdn.example/1/original.jpg", 2]]); // fail twice, then succeed + const adapter = fakeAdapter( + [{ data: [imageBlock(1)], meta: meta({ total_count: 1 }) }], + { failUrls }, + ); + + const summary = await executeDownload({ + ...baseOptions, + directory: dir, + adapter, + }); + assert.equal(summary.downloaded, 1); + assert.equal(summary.failed, 0); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("executeDownload skips existing files unless overwrite is set", async () => { + const dir = await mkdtemp(join(tmpdir(), "arena-dl-")); + try { + await writeFile(join(dir, "0001_photo-1.jpg"), "preexisting"); + const opened: string[] = []; + const adapter = fakeAdapter( + [{ data: [imageBlock(1)], meta: meta({ total_count: 1 }) }], + { onOpen: (url) => opened.push(url) }, + ); + + const summary = await executeDownload({ + ...baseOptions, + directory: dir, + adapter, + }); + assert.equal(summary.skipped, 1); + assert.equal(summary.downloaded, 0); + assert.equal( + opened.length, + 0, + "should not fetch an asset that already exists", + ); + + // Untouched on disk. + assert.equal( + await readFile(join(dir, "0001_photo-1.jpg"), "utf-8"), + "preexisting", + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("executeDownload filters by --type", async () => { + const dir = await mkdtemp(join(tmpdir(), "arena-dl-")); + try { + const adapter = fakeAdapter([ + { + data: [imageBlock(1), attachmentBlock(2)], + meta: meta({ total_count: 2 }), + }, + ]); + + const summary = await executeDownload({ + ...baseOptions, + type: "Attachment", + directory: dir, + adapter, + }); + + assert.equal(summary.listed, 2); + assert.equal(summary.downloaded, 1); // only the attachment + const files = (await readdir(dir)).filter((f) => f !== "manifest.json"); + assert.deepEqual(files, ["0002_doc-2.pdf"]); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/src/lib/download.ts b/src/lib/download.ts new file mode 100644 index 0000000..1cdaafe --- /dev/null +++ b/src/lib/download.ts @@ -0,0 +1,435 @@ +import { createWriteStream } from "node:fs"; +import { access, mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { ArenaError, client, getData } from "../api/client"; +import type { Block, Connectable, PaginationMeta } from "../api/types"; +import { cancellationSignal, fetchWithTimeout } from "./network"; + +export type ImageSize = "original" | "large" | "medium" | "small" | "square"; + +export interface DownloadOptions { + slug: string; + directory?: string; + size: ImageSize; + concurrency: number; + includeText: boolean; + type?: string; + overwrite: boolean; +} + +export interface DownloadFailure { + file: string; + error: string; +} + +export interface ManifestEntry { + id: number; + type: string; + title: string | null; + filename: string | null; + source_url: string | null; + description: string | null; + created_at: string; +} + +export interface DownloadSummary { + channel: string; + directory: string; + listed: number; + downloaded: number; + skipped: number; + failed: number; + failures: DownloadFailure[]; +} + +export type DownloadEvent = + | { type: "list_started"; slug: string } + | { type: "list_progress"; fetched: number; total: number } + | { type: "list_completed"; total: number; downloadable: number } + | { + type: "file_progress"; + index: number; + file: string; + completed: number; + total: number; + downloaded: number; + skipped: number; + failed: number; + } + | { type: "file_failed"; index: number; file: string; error: string } + | { type: "completed"; summary: DownloadSummary }; + +/** A unit of work: either a remote asset to fetch or local text to write. */ +export type DownloadTarget = + | { kind: "fetch"; url: string; filename: string } + | { kind: "text"; content: string; filename: string }; + +/** + * Injectable seam for network access, mirroring the import command's adapter. + * The default implementation hits the real API/CDN; tests substitute their own. + */ +export interface DownloadAdapter { + listContents( + slug: string, + page: number, + per: number, + ): Promise<{ data: Connectable[]; meta: PaginationMeta }>; + /** Open a remote asset for reading, throwing on a non-OK response. */ + openAsset(url: string, signal: AbortSignal): Promise; + sleep(ms: number): Promise; +} + +const MAX_DOWNLOAD_ATTEMPTS = 3; +const RETRY_BASE_DELAY_MS = 300; +const PER_PAGE = 100; + +const CONTENT_TYPE_EXTENSIONS: Record = { + "image/jpeg": "jpg", + "image/jpg": "jpg", + "image/png": "png", + "image/gif": "gif", + "image/webp": "webp", + "image/svg+xml": "svg", + "image/tiff": "tiff", + "application/pdf": "pdf", +}; + +function throwIfAborted(signal?: AbortSignal): void { + if (!signal?.aborted) return; + const reason = signal.reason; + if (reason instanceof Error) throw reason; + throw new Error("Operation cancelled by user"); +} + +function isTransientError(err: unknown): boolean { + if (err instanceof ArenaError) { + return err.status >= 500 || err.status === 429; + } + const message = + err instanceof Error ? err.message.toLowerCase() : String(err); + return ( + message.includes("timed out") || + message.includes("fetch failed") || + message.includes("network") || + message.includes("econnreset") || + message.includes("socket") + ); +} + +async function retryWithBackoff( + run: () => Promise, + sleep: (ms: number) => Promise, +): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= MAX_DOWNLOAD_ATTEMPTS; attempt++) { + try { + return await run(); + } catch (err: unknown) { + lastError = err; + if (attempt >= MAX_DOWNLOAD_ATTEMPTS || !isTransientError(err)) throw err; + await sleep(Math.min(RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), 5_000)); + } + } + throw lastError instanceof Error ? lastError : new Error(String(lastError)); +} + +/** Strip path separators and control characters so a block filename is safe to join. */ +function sanitizeFilename(name: string): string { + return name + .replace(/[/\\]/g, "_") + .replace(/[\x00-\x1f]/g, "") + .trim(); +} + +function extensionForContentType(contentType?: string | null): string { + if (!contentType) return "bin"; + return CONTENT_TYPE_EXTENSIONS[contentType.toLowerCase()] ?? "bin"; +} + +async function emit( + onEvent: ((event: DownloadEvent) => void | Promise) | undefined, + event: DownloadEvent, +): Promise { + if (onEvent) await onEvent(event); +} + +/** Page through a channel's contents, accumulating every block. */ +async function listChannelBlocks( + slug: string, + adapter: DownloadAdapter, + onEvent: ((event: DownloadEvent) => void | Promise) | undefined, + signal?: AbortSignal, +): Promise { + const blocks: Block[] = []; + let page = 1; + + while (true) { + throwIfAborted(signal); + const response = await retryWithBackoff( + () => adapter.listContents(slug, page, PER_PAGE), + adapter.sleep, + ); + + // Channel contents can include nested channels; we only download blocks. + for (const item of response.data) { + if ("base_type" in item && item.base_type === "Block") blocks.push(item); + } + await emit(onEvent, { + type: "list_progress", + fetched: blocks.length, + total: response.meta.total_count, + }); + + if (!response.meta.has_more_pages || response.data.length === 0) break; + page = response.meta.next_page ?? page + 1; + } + + return blocks; +} + +/** Map a block to a downloadable target, or null if it has no local artifact. */ +export function resolveTarget( + block: Block, + index: number, + options: DownloadOptions, +): DownloadTarget | null { + const prefix = String(index + 1).padStart(4, "0"); + + switch (block.type) { + case "Image": { + const image = block.image; + const url = + options.size === "original" ? image.src : image[options.size]?.src; + if (!url) return null; + const base = + image.filename ?? + `${block.id}.${extensionForContentType(image.content_type)}`; + return { + kind: "fetch", + url, + filename: `${prefix}_${sanitizeFilename(base)}`, + }; + } + + case "Attachment": { + const attachment = block.attachment; + const base = + attachment.filename ?? + `${block.id}.${attachment.file_extension ?? extensionForContentType(attachment.content_type)}`; + return { + kind: "fetch", + url: attachment.url, + filename: `${prefix}_${sanitizeFilename(base)}`, + }; + } + + case "Text": { + if (!options.includeText) return null; + const content = block.content?.markdown ?? block.content?.plain ?? ""; + return { + kind: "text", + content, + filename: `${prefix}_${block.id}.md`, + }; + } + + default: + // Link, Embed, PendingBlock: no Are.na-hosted file. Captured in the manifest. + return null; + } +} + +export function buildManifest(blocks: Block[]): ManifestEntry[] { + return blocks.map((block) => ({ + id: block.id, + type: block.type, + title: block.title ?? null, + filename: + block.type === "Image" + ? (block.image.filename ?? null) + : block.type === "Attachment" + ? (block.attachment.filename ?? null) + : null, + source_url: block.source?.url ?? null, + description: block.description?.plain ?? null, + created_at: block.created_at, + })); +} + +async function writeTarget( + target: DownloadTarget, + destDir: string, + overwrite: boolean, + adapter: DownloadAdapter, + signal: AbortSignal, +): Promise<"written" | "skipped"> { + const destPath = join(destDir, target.filename); + + if (!overwrite) { + const exists = await access(destPath).then( + () => true, + () => false, + ); + if (exists) return "skipped"; + } + + if (target.kind === "text") { + await writeFile(destPath, target.content, "utf-8"); + return "written"; + } + + const source = await adapter.openAsset(target.url, signal); + await pipeline(source, createWriteStream(destPath)); + return "written"; +} + +/** Real adapter: paginated API listing + a CDN fetch that streams to disk. */ +export function defaultDownloadAdapter(): DownloadAdapter { + return { + async listContents(slug, page, per) { + return getData( + client.GET("/v3/channels/{id}/contents", { + params: { + path: { id: slug }, + query: { page, per, sort: "position_desc" }, + }, + }), + ); + }, + async openAsset(url, signal) { + // Are.na's CloudFront CDN returns 202 + empty body when no User-Agent is + // sent, silently producing 0-byte files. Send one explicitly. + const response = await fetchWithTimeout(url, { + signal, + headers: { "User-Agent": "@aredotna/cli (arena download)" }, + }); + if (!response.ok || !response.body) { + throw new Error(`HTTP ${response.status} fetching ${url}`); + } + return Readable.fromWeb( + response.body as Parameters[0], + ); + }, + async sleep(ms) { + await new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); + }, + }; +} + +export interface ExecuteDownloadOptions extends DownloadOptions { + adapter?: DownloadAdapter; + onEvent?: (event: DownloadEvent) => void | Promise; + signal?: AbortSignal; +} + +export function downloadExitCode(summary: DownloadSummary): number { + return summary.failed > 0 ? 1 : 0; +} + +export async function executeDownload( + options: ExecuteDownloadOptions, +): Promise { + const { slug, onEvent } = options; + const adapter = options.adapter ?? defaultDownloadAdapter(); + const signal = options.signal ?? cancellationSignal(); + + throwIfAborted(signal); + await emit(onEvent, { type: "list_started", slug }); + + const blocks = await listChannelBlocks(slug, adapter, onEvent, signal); + + const destDir = resolve(options.directory ?? slug); + await mkdir(destDir, { recursive: true }); + + const typeFilter = options.type?.toLowerCase(); + const targets: Array<{ index: number; target: DownloadTarget }> = []; + blocks.forEach((block, index) => { + if (typeFilter && block.type.toLowerCase() !== typeFilter) return; + const target = resolveTarget(block, index, options); + if (target) targets.push({ index, target }); + }); + + await emit(onEvent, { + type: "list_completed", + total: blocks.length, + downloadable: targets.length, + }); + + // Always write a manifest so Link/Embed/Text metadata is preserved. + await writeFile( + join(destDir, "manifest.json"), + JSON.stringify(buildManifest(blocks), null, 2), + "utf-8", + ); + + const failures: DownloadFailure[] = []; + let completed = 0; + let downloaded = 0; + let skipped = 0; + let failed = 0; + + let cursor = 0; + const workerCount = Math.max( + 1, + Math.min(options.concurrency, Math.max(1, targets.length)), + ); + + const workers = Array.from({ length: workerCount }, async () => { + while (true) { + throwIfAborted(signal); + const current = cursor; + cursor += 1; + if (current >= targets.length) return; + + const { target } = targets[current]!; + try { + const result = await retryWithBackoff( + () => + writeTarget(target, destDir, options.overwrite, adapter, signal), + adapter.sleep, + ); + if (result === "written") downloaded += 1; + else skipped += 1; + } catch (err: unknown) { + const error = err instanceof Error ? err.message : String(err); + failures.push({ file: target.filename, error }); + failed += 1; + await emit(onEvent, { + type: "file_failed", + index: current, + file: target.filename, + error, + }); + } finally { + completed += 1; + await emit(onEvent, { + type: "file_progress", + index: current, + file: target.filename, + completed, + total: targets.length, + downloaded, + skipped, + failed, + }); + } + } + }); + + await Promise.all(workers); + + const summary: DownloadSummary = { + channel: slug, + directory: destDir, + listed: blocks.length, + downloaded, + skipped, + failed, + failures, + }; + + await emit(onEvent, { type: "completed", summary }); + return summary; +} diff --git a/src/lib/registry.tsx b/src/lib/registry.tsx index 6a697fa..15e049c 100644 --- a/src/lib/registry.tsx +++ b/src/lib/registry.tsx @@ -80,6 +80,11 @@ import { runImportJsonStream, } from "../commands/import"; import { FeedCommand } from "../commands/feed"; +import { + DownloadCommand, + parseDownloadOptions, + runDownloadJsonStream, +} from "../commands/download"; import { PingCommand } from "../commands/ping"; import { SearchCommand } from "../commands/search"; import { @@ -379,6 +384,36 @@ export const commands: CommandDefinition[] = [ }, }, + { + name: "download", + aliases: ["dl"], + group: "Channels", + help: [ + { + usage: + "download [--dir ] [--size ] [--concurrency ] [--include-text] [--type ] [--overwrite]", + description: "Options", + }, + { + usage: "download worldmaking --dir ./refs", + description: "Example", + }, + { + usage: "download my-private-channel --type Image --size large", + description: "Example", + }, + ], + session: { args: "", desc: "Download a channel's files" }, + render(args, flags) { + return ; + }, + async jsonStream(args, flags, write) { + return runDownloadJsonStream(parseDownloadOptions(args, flags), (event) => + write(event), + ); + }, + }, + { name: "block", aliases: ["bl"], @@ -1940,6 +1975,45 @@ export const commandHelpDocs: Record = { }, seeAlso: ["search", "add", "connect"], }, + download: { + summary: + "Download a channel's images and attachments to a local directory. Works with private channels when authenticated.", + usage: ["arena download [flags]"], + options: [ + { + flag: "--dir ", + description: "Output directory (default: the channel slug)", + }, + { + flag: "--size ", + description: "Image resolution to download (default: original)", + }, + { + flag: "--concurrency ", + description: "Concurrent downloads (default: 4)", + }, + { + flag: "--include-text", + description: "Also write Text blocks as .md files", + }, + { + flag: "--type ", + description: "Only download blocks of this type", + }, + { + flag: "--overwrite", + description: "Re-download files that already exist locally", + }, + ], + examples: [ + "arena download worldmaking --dir ./refs", + "arena download my-private-channel --type Image --size large", + ], + notes: [ + "A manifest.json with block metadata (including Link/Embed source URLs) is always written.", + ], + seeAlso: ["channel", "import", "upload"], + }, block: { summary: "View and manage blocks.", usage: ["arena block ", "arena block ..."],