Class UICaptureService

Hierarchy

{
    captureWebsite: (
        url: string,
    ) => Effect<
        Map<string, CaptureResult>,
        BrowserError | CaptureError | FileSystemError,
    >;
} & { _tag: "UICaptureService" }
- UICaptureService

Index

Constructors

constructor

new UICaptureService(
    _: {
        captureWebsite: (
            url: string,
        ) => Effect<
            Map<string, CaptureResult>,
            BrowserError | CaptureError | FileSystemError,
        >;
    },
): UICaptureService
Parameters
- _: {
      captureWebsite: (
          url: string,
      ) => Effect<
          Map<string, CaptureResult>,
          BrowserError | CaptureError | FileSystemError,
      >;
  }
Returns UICaptureService
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).constructor
- Defined in node_modules/effect/dist/dts/Effect.d.ts:26568

Properties

captureWebsite

captureWebsite: (
    url: string,
) => Effect<
    Map<string, CaptureResult>,
    BrowserError | CaptureError | FileSystemError,
>

Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).captureWebsite

Defined in src/service.ts:490

`Static` `Readonly`[ChannelTypeId]

"[ChannelTypeId]": VarianceStruct<
    never,
    unknown,
    never,
    unknown,
    UICaptureService,
    unknown,
    UICaptureService,
>

Defined in node_modules/effect/dist/dts/Channel.d.ts:108

`Static` `Readonly`_op

_op: "Tag"

Defined in node_modules/effect/dist/dts/Context.d.ts:33

`Static` `Readonly`Service

Service: UICaptureService

Defined in node_modules/effect/dist/dts/Context.d.ts:34

`Static` `Readonly`Identifier

Identifier: UICaptureService

Defined in node_modules/effect/dist/dts/Context.d.ts:35

`Static` `Readonly`[TagTypeId]

"[TagTypeId]": {
_Service: Invariant<UICaptureService>;
_Identifier: Invariant<UICaptureService>;
}

Defined in node_modules/effect/dist/dts/Context.d.ts:36

`Static` `Optional` `Readonly`stack

stack?: string

Defined in node_modules/effect/dist/dts/Context.d.ts:42

`Static` `Readonly`key

key: "UICaptureService"

Defined in node_modules/effect/dist/dts/Context.d.ts:43
Defined in node_modules/effect/dist/dts/Effect.d.ts:26574

`Static` `Optional`[typeSymbol]

"[typeSymbol]"?: unknown

Defined in node_modules/effect/dist/dts/Context.d.ts:44

`Static` `Optional`[unifySymbol]

"[unifySymbol]"?: TagUnify<
    Class<
        UICaptureService,
        "UICaptureService",
        {
            effect: Effect<
                {
                    captureWebsite: (
                        url: string,
                    ) => Effect<
                        Map<string, CaptureResult>,
                        BrowserError | CaptureError | FileSystemError,
                    >;
                },
                never,
                CaptureConfigTag,
            >;
        },
    >,
>

Defined in node_modules/effect/dist/dts/Context.d.ts:45

`Static` `Optional`[ignoreSymbol]

"[ignoreSymbol]"?: TagUnifyIgnore

Defined in node_modules/effect/dist/dts/Context.d.ts:46

`Static` `Readonly`[EffectTypeId]

"[EffectTypeId]": VarianceStruct<UICaptureService, never, UICaptureService>

Defined in node_modules/effect/dist/dts/Effect.d.ts:195

`Readonly`_tag

_tag: "UICaptureService"

Defined in node_modules/effect/dist/dts/Effect.d.ts:26569

`Static` `Readonly`use

use: <X>(
    body: (_: UICaptureService) => X,
) => [X] extends [Effect<A, E, R>]
    ? Effect<A, E, UICaptureService | R>
    : [X] extends [PromiseLike<A>]
        ? Effect<A, UnknownException, UICaptureService>
        : Effect<X, never, UICaptureService>

Defined in node_modules/effect/dist/dts/Effect.d.ts:26571

`Static` `Readonly`make

make: (
    _: {
        captureWebsite: (
            url: string,
        ) => Effect<
            Map<string, CaptureResult>,
            BrowserError | CaptureError | FileSystemError,
        >;
    },
) => UICaptureService

Defined in node_modules/effect/dist/dts/Effect.d.ts:26572

`Static` `Readonly`Default

Default: Layer<UICaptureService, never, CaptureConfigTag>

Defined in node_modules/effect/dist/dts/Effect.d.ts:26576

`Static` `Readonly`[STMTypeId]

"[STMTypeId]": {
    _A: Covariant<UICaptureService>;
    _E: Covariant<never>;
    _R: Covariant<UICaptureService>;
}

Defined in node_modules/effect/dist/dts/STM.d.ts:136

`Static` `Readonly`[SinkTypeId]

"[SinkTypeId]": VarianceStruct<
    UICaptureService,
    unknown,
    never,
    never,
    UICaptureService,
>

Defined in node_modules/effect/dist/dts/Sink.d.ts:82

`Static` `Readonly`[StreamTypeId]

"[StreamTypeId]": VarianceStruct<UICaptureService, never, UICaptureService>

Defined in node_modules/effect/dist/dts/Stream.d.ts:111

Methods

`Static`of

of(self: UICaptureService): UICaptureService
Parameters
- self: UICaptureService
Returns UICaptureService
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).of
- Defined in node_modules/effect/dist/dts/Context.d.ts:40

`Static`context

context(self: UICaptureService): Context<UICaptureService>
Parameters
- self: UICaptureService
Returns Context<UICaptureService>
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).context
- Defined in node_modules/effect/dist/dts/Context.d.ts:41

`Static`[iterator]

"[iterator]"(): EffectGenerator<Tag<UICaptureService, UICaptureService>>
Returns EffectGenerator<Tag<UICaptureService, UICaptureService>>
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).[iterator]
- Defined in node_modules/effect/dist/dts/Effect.d.ts:137

`Static`toString

toString(): string
Returns string
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).toString
- Defined in node_modules/effect/dist/dts/Inspectable.d.ts:20

`Static`toJSON

toJSON(): unknown
Returns unknown
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).toJSON
- Defined in node_modules/effect/dist/dts/Inspectable.d.ts:21

`Static`[NodeInspectSymbol]

"[NodeInspectSymbol]"(): unknown
Returns unknown
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).[NodeInspectSymbol]
- Defined in node_modules/effect/dist/dts/Inspectable.d.ts:22

`Static`pipe

pipe<A>(this: A): A
Type Parameters
- A
Parameters
- this: A
Returns A
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:10
pipe<A, B = never>(this: A, ab: (_: A) => B): B
Type Parameters
- A
- B = never
Parameters
- this: A
- ab: (_: A) => B
Returns B
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:11
pipe<A, B = never, C = never>(this: A, ab: (_: A) => B, bc: (_: B) => C): C
Type Parameters
- A
- B = never
- C = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
Returns C
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:12
pipe<A, B = never, C = never, D = never>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
): D
Type Parameters
- A
- B = never
- C = never
- D = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
Returns D
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:13
pipe<A, B = never, C = never, D = never, E = never>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
): E
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
Returns E
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:14
pipe<A, B = never, C = never, D = never, E = never, F = never>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
): F
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
Returns F
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:15
pipe<A, B = never, C = never, D = never, E = never, F = never, G = never>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
): G
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
Returns G
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:16
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
): H
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
Returns H
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:17
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
): I
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
Returns I
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:18
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
    J = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
    ij: (_: I) => J,
): J
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
- J = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
- ij: (_: I) => J
Returns J
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:19
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
    J = never,
    K = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
    ij: (_: I) => J,
    jk: (_: J) => K,
): K
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
- J = never
- K = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
- ij: (_: I) => J
- jk: (_: J) => K
Returns K
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:20
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
    J = never,
    K = never,
    L = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
    ij: (_: I) => J,
    jk: (_: J) => K,
    kl: (_: K) => L,
): L
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
- J = never
- K = never
- L = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
- ij: (_: I) => J
- jk: (_: J) => K
- kl: (_: K) => L
Returns L
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:21
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
    J = never,
    K = never,
    L = never,
    M = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
    ij: (_: I) => J,
    jk: (_: J) => K,
    kl: (_: K) => L,
    lm: (_: L) => M,
): M
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
- J = never
- K = never
- L = never
- M = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
- ij: (_: I) => J
- jk: (_: J) => K
- kl: (_: K) => L
- lm: (_: L) => M
Returns M
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:22
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
    J = never,
    K = never,
    L = never,
    M = never,
    N = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
    ij: (_: I) => J,
    jk: (_: J) => K,
    kl: (_: K) => L,
    lm: (_: L) => M,
    mn: (_: M) => N,
): N
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
- J = never
- K = never
- L = never
- M = never
- N = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
- ij: (_: I) => J
- jk: (_: J) => K
- kl: (_: K) => L
- lm: (_: L) => M
- mn: (_: M) => N
Returns N
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:23
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
    J = never,
    K = never,
    L = never,
    M = never,
    N = never,
    O = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
    ij: (_: I) => J,
    jk: (_: J) => K,
    kl: (_: K) => L,
    lm: (_: L) => M,
    mn: (_: M) => N,
    no: (_: N) => O,
): O
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
- J = never
- K = never
- L = never
- M = never
- N = never
- O = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
- ij: (_: I) => J
- jk: (_: J) => K
- kl: (_: K) => L
- lm: (_: L) => M
- mn: (_: M) => N
- no: (_: N) => O
Returns O
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:24
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
    J = never,
    K = never,
    L = never,
    M = never,
    N = never,
    O = never,
    P = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
    ij: (_: I) => J,
    jk: (_: J) => K,
    kl: (_: K) => L,
    lm: (_: L) => M,
    mn: (_: M) => N,
    no: (_: N) => O,
    op: (_: O) => P,
): P
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
- J = never
- K = never
- L = never
- M = never
- N = never
- O = never
- P = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
- ij: (_: I) => J
- jk: (_: J) => K
- kl: (_: K) => L
- lm: (_: L) => M
- mn: (_: M) => N
- no: (_: N) => O
- op: (_: O) => P
Returns P
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:25
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
    J = never,
    K = never,
    L = never,
    M = never,
    N = never,
    O = never,
    P = never,
    Q = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
    ij: (_: I) => J,
    jk: (_: J) => K,
    kl: (_: K) => L,
    lm: (_: L) => M,
    mn: (_: M) => N,
    no: (_: N) => O,
    op: (_: O) => P,
    pq: (_: P) => Q,
): Q
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
- J = never
- K = never
- L = never
- M = never
- N = never
- O = never
- P = never
- Q = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
- ij: (_: I) => J
- jk: (_: J) => K
- kl: (_: K) => L
- lm: (_: L) => M
- mn: (_: M) => N
- no: (_: N) => O
- op: (_: O) => P
- pq: (_: P) => Q
Returns Q
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:26
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
    J = never,
    K = never,
    L = never,
    M = never,
    N = never,
    O = never,
    P = never,
    Q = never,
    R = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
    ij: (_: I) => J,
    jk: (_: J) => K,
    kl: (_: K) => L,
    lm: (_: L) => M,
    mn: (_: M) => N,
    no: (_: N) => O,
    op: (_: O) => P,
    pq: (_: P) => Q,
    qr: (_: Q) => R,
): R
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
- J = never
- K = never
- L = never
- M = never
- N = never
- O = never
- P = never
- Q = never
- R = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
- ij: (_: I) => J
- jk: (_: J) => K
- kl: (_: K) => L
- lm: (_: L) => M
- mn: (_: M) => N
- no: (_: N) => O
- op: (_: O) => P
- pq: (_: P) => Q
- qr: (_: Q) => R
Returns R
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:27
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
    J = never,
    K = never,
    L = never,
    M = never,
    N = never,
    O = never,
    P = never,
    Q = never,
    R = never,
    S = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
    ij: (_: I) => J,
    jk: (_: J) => K,
    kl: (_: K) => L,
    lm: (_: L) => M,
    mn: (_: M) => N,
    no: (_: N) => O,
    op: (_: O) => P,
    pq: (_: P) => Q,
    qr: (_: Q) => R,
    rs: (_: R) => S,
): S
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
- J = never
- K = never
- L = never
- M = never
- N = never
- O = never
- P = never
- Q = never
- R = never
- S = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
- ij: (_: I) => J
- jk: (_: J) => K
- kl: (_: K) => L
- lm: (_: L) => M
- mn: (_: M) => N
- no: (_: N) => O
- op: (_: O) => P
- pq: (_: P) => Q
- qr: (_: Q) => R
- rs: (_: R) => S
Returns S
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:28
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
    J = never,
    K = never,
    L = never,
    M = never,
    N = never,
    O = never,
    P = never,
    Q = never,
    R = never,
    S = never,
    T = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
    ij: (_: I) => J,
    jk: (_: J) => K,
    kl: (_: K) => L,
    lm: (_: L) => M,
    mn: (_: M) => N,
    no: (_: N) => O,
    op: (_: O) => P,
    pq: (_: P) => Q,
    qr: (_: Q) => R,
    rs: (_: R) => S,
    st: (_: S) => T,
): T
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
- J = never
- K = never
- L = never
- M = never
- N = never
- O = never
- P = never
- Q = never
- R = never
- S = never
- T = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
- ij: (_: I) => J
- jk: (_: J) => K
- kl: (_: K) => L
- lm: (_: L) => M
- mn: (_: M) => N
- no: (_: N) => O
- op: (_: O) => P
- pq: (_: P) => Q
- qr: (_: Q) => R
- rs: (_: R) => S
- st: (_: S) => T
Returns T
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:29
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
    J = never,
    K = never,
    L = never,
    M = never,
    N = never,
    O = never,
    P = never,
    Q = never,
    R = never,
    S = never,
    T = never,
    U = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
    ij: (_: I) => J,
    jk: (_: J) => K,
    kl: (_: K) => L,
    lm: (_: L) => M,
    mn: (_: M) => N,
    no: (_: N) => O,
    op: (_: O) => P,
    pq: (_: P) => Q,
    qr: (_: Q) => R,
    rs: (_: R) => S,
    st: (_: S) => T,
    tu: (_: T) => U,
): U
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
- J = never
- K = never
- L = never
- M = never
- N = never
- O = never
- P = never
- Q = never
- R = never
- S = never
- T = never
- U = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
- ij: (_: I) => J
- jk: (_: J) => K
- kl: (_: K) => L
- lm: (_: L) => M
- mn: (_: M) => N
- no: (_: N) => O
- op: (_: O) => P
- pq: (_: P) => Q
- qr: (_: Q) => R
- rs: (_: R) => S
- st: (_: S) => T
- tu: (_: T) => U
Returns U
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:30
pipe<
    A,
    B = never,
    C = never,
    D = never,
    E = never,
    F = never,
    G = never,
    H = never,
    I = never,
    J = never,
    K = never,
    L = never,
    M = never,
    N = never,
    O = never,
    P = never,
    Q = never,
    R = never,
    S = never,
    T = never,
    U = never,
>(
    this: A,
    ab: (_: A) => B,
    bc: (_: B) => C,
    cd: (_: C) => D,
    de: (_: D) => E,
    ef: (_: E) => F,
    fg: (_: F) => G,
    gh: (_: G) => H,
    hi: (_: H) => I,
    ij: (_: I) => J,
    jk: (_: J) => K,
    kl: (_: K) => L,
    lm: (_: L) => M,
    mn: (_: M) => N,
    no: (_: N) => O,
    op: (_: O) => P,
    pq: (_: P) => Q,
    qr: (_: Q) => R,
    rs: (_: R) => S,
    st: (_: S) => T,
    tu: (_: T) => U,
): U
Type Parameters
- A
- B = never
- C = never
- D = never
- E = never
- F = never
- G = never
- H = never
- I = never
- J = never
- K = never
- L = never
- M = never
- N = never
- O = never
- P = never
- Q = never
- R = never
- S = never
- T = never
- U = never
Parameters
- this: A
- ab: (_: A) => B
- bc: (_: B) => C
- cd: (_: C) => D
- de: (_: D) => E
- ef: (_: E) => F
- fg: (_: F) => G
- gh: (_: G) => H
- hi: (_: H) => I
- ij: (_: I) => J
- jk: (_: J) => K
- kl: (_: K) => L
- lm: (_: L) => M
- mn: (_: M) => N
- no: (_: N) => O
- op: (_: O) => P
- pq: (_: P) => Q
- qr: (_: Q) => R
- rs: (_: R) => S
- st: (_: S) => T
- tu: (_: T) => U
Returns U
Inherited from Effect.Service<UICaptureService>()( "UICaptureService", { effect: Effect.gen(function* () { const cfg = yield* CaptureConfigTag; let browser: Browser | null = null; const processedRoutes = new Set<string>(); const hostFilters = createHostFilterState(); const initialize = Effect.tryPromise({ try: async () => { await fs.mkdir(cfg.outputDir, { recursive: true }); browser = await chromium.launch({ headless: true, args: [ "--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", ], }); console.log("✓ Browser initialized"); return browser; }, catch: (error) => new BrowserError({ message: "Failed to initialize browser", cause: error, }), }); const cleanup = Effect.tryPromise({ try: async () => { if (browser) { await browser.close(); browser = null; } processedRoutes.clear(); console.log("✓ Browser cleanup complete"); }, catch: (error) => new BrowserError({ message: "Failed to cleanup browser", cause: error, }), }); const { prepareForLinkDiscovery, extractLinks } = createLinkDiscoveryTools({ hostMatchesFilters: (hostname) => hostFilters.hostMatchesFilters(hostname, cfg.includeSubdomains), menuInteractionSelectors: cfg.menuInteractionSelectors, }); const capturePage = ( page: Page, url: string, ): Effect.Effect<CaptureResult, CaptureError | FileSystemError> => Effect.gen(function* () { const route = getRouteName(url); const routeDir = path.join(cfg.outputDir, route); yield* createDirectories(routeDir, cfg.captureVideo); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle"), catch: (error) => new CaptureError({ url, message: "Failed to wait for page load", cause: error, }), }); yield* Effect.sleep(cfg.waitTime); if (cfg.warmupScroll) { yield* performWarmupScroll(page, url).pipe( Effect.catchAll((error) => { console.warn( ` ! Warm-up scroll failed (continuing): ${error.message}`, ); return Effect.void; }), ); yield* Effect.tryPromise({ try: () => page.waitForLoadState("networkidle", { timeout: 10000 }), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)); } const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); const screenshotResults = yield* Effect.all( cfg.viewports.map((viewport: ViewportConfig) => Effect.gen(function* () { console.log( ` Capturing ${viewport.name} (${viewport.width}x${viewport.height})`, ); const screenshots = yield* captureScreenshots( page, viewport, routeDir, timestamp, { ffmpegPath: cfg.ffmpegPath, screenshotHideSelectors: cfg.screenshotHideSelectors, }, ); const videos = cfg.captureVideo && browser ? Option.some( yield* captureVideoForViewport( browser, page, viewport, routeDir, timestamp, { waitTime: cfg.waitTime, ffmpegPath: cfg.ffmpegPath, videoOptions: cfg.videoOptions, }, ), ) : Option.none(); return [viewport.name, { screenshots, videos }] as const; }), ), { concurrency: 1 }, ); const screenshots: Record<string, ScreenshotPaths> = {}; const videos: Record<string, VideoQualityPaths> = {}; for (const [name, data] of screenshotResults) { screenshots[name] = data.screenshots; if (Option.isSome(data.videos)) { videos[name] = data.videos.value; } } return new CaptureResult({ url, route, screenshots, videos: Object.keys(videos).length > 0 ? videos : undefined, timestamp: Date.now(), }); }); const processRouteTask = ( page: Page, task: RouteTask, results: Map<string, CaptureResult>, scheduleNext: ( url: string, depth: number, ) => Effect.Effect<void, never>, workerLabel: string, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { const indent = " ".repeat(task.depth); console.log( `\n${indent}[Worker ${workerLabel}] [Depth ${task.depth}] Capturing: ${task.url}`, ); yield* Effect.tryPromise({ try: () => page.goto(task.url, { waitUntil: "networkidle", timeout: 30000 }), catch: (error) => new CaptureError({ url: task.url, message: "Failed to navigate", cause: error, }), }).pipe(Effect.retry(navigationRetryPolicy)); yield* prepareForLinkDiscovery(page, task.url); const discoveredLinks = task.depth < cfg.maxDepth ? yield* extractLinks(page) : []; console.log( `${indent} Found ${discoveredLinks.length} internal links`, ); const result = yield* capturePage(page, task.url); results.set(task.normalizedUrl, result); if (discoveredLinks.length > 0) { const schedulingConcurrency = Math.max( 1, Math.min(cfg.routeConcurrency, discoveredLinks.length), ); yield* Effect.forEach( discoveredLinks, (link) => scheduleNext(link, task.depth + 1), { concurrency: schedulingConcurrency }, ); } }); const captureWebsite = ( url: string, ): Effect.Effect< Map<string, CaptureResult>, BrowserError | CaptureError | FileSystemError > => Effect.gen(function* () { console.log("Starting UI capture for:", url); const urlObj = new URL(url); hostFilters.hydrate(urlObj.hostname, cfg.allowedHosts); const results = new Map<string, CaptureResult>(); yield* Effect.acquireUseRelease( initialize, () => Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const queueCapacity = Math.max(32, cfg.routeConcurrency * 8); const taskQueue = yield* Queue.bounded<QueueTask>(queueCapacity); const pendingTasks = yield* Ref.make(0); const shutdownNotified = yield* Ref.make(false); const signalShutdown = (): Effect.Effect<void, never> => Effect.gen(function* () { const already = yield* Ref.get(shutdownNotified); if (already) return; yield* Ref.set(shutdownNotified, true); for (let i = 0; i < cfg.routeConcurrency; i++) { yield* Queue.offer(taskQueue, ShutdownSignal); } }); const scheduleRoute = ( routeUrl: string, depth: number, ): Effect.Effect<void, never> => Effect.gen(function* () { if (depth > cfg.maxDepth) return; if (yield* Ref.get(shutdownNotified)) return; let hostname: string; try { hostname = new URL(routeUrl).hostname; } catch { return; } if ( !hostFilters.hostMatchesFilters( hostname, cfg.includeSubdomains, ) ) { return; } const normalizedUrlStr = normalizeUrl(routeUrl); const taskOption = yield* Effect.sync(() => { if (processedRoutes.has(normalizedUrlStr)) { return Option.none<RouteTask>(); } processedRoutes.add(normalizedUrlStr); return Option.some<RouteTask>({ type: "route", url: routeUrl, depth, normalizedUrl: normalizedUrlStr, }); }); if (Option.isSome(taskOption)) { yield* Ref.update(pendingTasks, (count) => count + 1); yield* Queue.offer(taskQueue, taskOption.value); } }); const markTaskComplete = (): Effect.Effect<void, never> => Effect.gen(function* () { const remaining = yield* Ref.updateAndGet( pendingTasks, (count) => Math.max(0, count - 1), ); if (remaining === 0) { yield* signalShutdown(); } }); const workerLoop = ( page: Page, workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.gen(function* () { while (true) { const task = yield* Queue.take(taskQueue); if (task.type === "shutdown") { return yield* Effect.void; } yield* processRouteTask( page, task, results, scheduleRoute, `#${workerId}`, ).pipe( Effect.catchAll((error) => { console.error( `[Worker ${workerId}] Failed to capture ${task.url}:`, error, ); return Effect.void; }), Effect.ensuring(markTaskComplete()), ); } }); const createWorker = ( workerId: number, ): Effect.Effect<void, CaptureError | FileSystemError> => Effect.acquireUseRelease( Effect.gen(function* () { if (!browser) { return yield* Effect.fail( new CaptureError({ url, message: "Browser not initialized", cause: null, }), ); } const browserRef = browser; const context = yield* Effect.tryPromise({ try: () => browserRef.newContext(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create context`, cause: error, }), }); const page = yield* Effect.tryPromise({ try: () => context.newPage(), catch: (error) => new CaptureError({ url, message: `Worker ${workerId}: Failed to create page`, cause: error, }), }); console.log(`✓ Worker ${workerId} ready`); return { context, page }; }), ({ page }) => workerLoop(page, workerId), ({ context }) => Effect.tryPromise({ try: () => context.close(), catch: () => undefined, }).pipe(Effect.catchAll(() => Effect.void)), ); yield* scheduleRoute(url, 0); const initialPending = yield* Ref.get(pendingTasks); if (initialPending === 0) { yield* signalShutdown(); } const workers = Array.from( { length: cfg.routeConcurrency }, (_, idx) => createWorker(idx + 1), ); yield* Effect.all(workers, { concurrency: cfg.routeConcurrency, }); yield* Queue.shutdown(taskQueue); yield* generateReports(cfg.outputDir, cfg.viewports, results); console.log( `\n✓ Capture completed! Results saved to: ${cfg.outputDir}`, ); }), () => cleanup.pipe(Effect.orDie), ); return results; }); return { captureWebsite } as const; }), }, ).pipe
- Defined in node_modules/effect/dist/dts/Pipeable.d.ts:31

Class UICaptureService

Hierarchy

Index

Constructors

Properties

Methods

Constructors

constructor

Parameters

Returns UICaptureService

Properties

captureWebsite

Static Readonly[ChannelTypeId]

Static Readonly_op

Static ReadonlyService

Static ReadonlyIdentifier

Static Readonly[TagTypeId]

Static Optional Readonlystack

Static Readonlykey

Static Optional[typeSymbol]

Static Optional[unifySymbol]

Static Optional[ignoreSymbol]

Static Readonly[EffectTypeId]

Readonly_tag

Static Readonlyuse

Static Readonlymake

Static ReadonlyDefault

Static Readonly[STMTypeId]

Static Readonly[SinkTypeId]

Static Readonly[StreamTypeId]

Methods

Staticof

Parameters

Returns UICaptureService

Staticcontext

Parameters

Returns Context<UICaptureService>

Static[iterator]

Returns EffectGenerator<Tag<UICaptureService, UICaptureService>>

StatictoString

Returns string

StatictoJSON

Returns unknown

Static[NodeInspectSymbol]

Returns unknown

Staticpipe

Type Parameters

Parameters

Returns A

Type Parameters

Parameters

Returns B

Type Parameters

Parameters

Returns C

Type Parameters

Parameters

Returns D

Type Parameters

Parameters

Returns E

Type Parameters

Parameters

Returns F

Type Parameters

Parameters

Returns G

Type Parameters

Parameters

Returns H

Type Parameters

Parameters

Returns I

Type Parameters

Parameters

Returns J

Type Parameters

Parameters

Returns K

Type Parameters

`Static` `Readonly`[ChannelTypeId]

`Static` `Readonly`_op

`Static` `Readonly`Service

`Static` `Readonly`Identifier

`Static` `Readonly`[TagTypeId]

`Static` `Optional` `Readonly`stack

`Static` `Readonly`key

`Static` `Optional`[typeSymbol]

`Static` `Optional`[unifySymbol]

`Static` `Optional`[ignoreSymbol]

`Static` `Readonly`[EffectTypeId]

`Readonly`_tag

`Static` `Readonly`use

`Static` `Readonly`make

`Static` `Readonly`Default

`Static` `Readonly`[STMTypeId]

`Static` `Readonly`[SinkTypeId]

`Static` `Readonly`[StreamTypeId]

`Static`of

`Static`context

`Static`[iterator]

`Static`toString

`Static`toJSON

`Static`[NodeInspectSymbol]

`Static`pipe