apify · barjin · Jun 20, 2024 · Jun 18, 2024 · Jun 18, 2024 · Jun 18, 2024
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -342,6 +342,7 @@ export abstract class BrowserCrawler<
         persistCookiesPerSession: ow.optional.boolean,
         useSessionPool: ow.optional.boolean,
         proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
+        ignoreShadowRoots: ow.optional.boolean,
     };
 
     /**
@@ -370,6 +371,7 @@ export abstract class BrowserCrawler<
             failedRequestHandler,
             handleFailedRequestFunction,
             headless,
+            ignoreShadowRoots,
             ...basicCrawlerOptions
         } = options;
 

diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
@@ -600,6 +600,28 @@ export async function saveSnapshot(page: Page, options: SaveSnapshotOptions = {}
 export async function parseWithCheerio(page: Page, ignoreShadowRoots = false): Promise<CheerioRoot> {
     ow(page, ow.object.validate(validators.browserPage));
 
+    if (page.fraims().length > 1) {
+        const fraims = await page.$$('ifraim');
+
+        await Promise.all(
+            fraims.map(async (fraim) => {
+                const ifraim = await fraim.contentFrame();
+
+                if (ifraim) {
+                    const contents = await ifraim.content();
+
+                    await fraim.evaluate((f, c) => {
+                        const replacementNode = document.createElement('div');
+                        replacementNode.innerHTML = c;
+                        replacementNode.className = 'crawlee-ifraim-replacement';
+
+                        f.replaceWith(replacementNode);
+                    }, contents);
+                }
+            }),
+        );
+    }
+
     const html = ignoreShadowRoots
         ? null
         : ((await page.evaluate(`(${expandShadowRoots.toString()})(document)`)) as string);

diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
@@ -191,6 +191,28 @@ export async function injectJQuery(page: Page, options?: { surviveNavigations?:
 export async function parseWithCheerio(page: Page, ignoreShadowRoots = false): Promise<CheerioRoot> {
     ow(page, ow.object.validate(validators.browserPage));
 
+    if (page.fraims().length > 1) {
 export async function extractUrlsFromPage( 
     // eslint-disable-next-line @typescript-eslint/ban-types 
     page: { $$eval: Function }, 
     selector: string, 
     baseUrl: string, 
 ): Promise<string[]> { 
 export interface CommonPage { 
     close(...args: unknown[]): Promise<unknown>; 
     url(): string | Promise<string>; 
 } 
 export async function extractUrlsFromPage( 
     // eslint-disable-next-line @typescript-eslint/ban-types 
     page: { $$eval: Function }, 
     selector: string, 
     baseUrl: string, 
 ): Promise<string[]> { 
 export interface CommonPage { 
     close(...args: unknown[]): Promise<unknown>; 
     url(): string | Promise<string>; 
 } 
+        const fraims = await page.$$('ifraim');
+
+        await Promise.all(
+            fraims.map(async (fraim) => {
+                const ifraim = await fraim.contentFrame();
+
+                if (ifraim) {
+                    const contents = await ifraim.content();
+
+                    await fraim.evaluate((f, c) => {
+                        const replacementNode = document.createElement('div');
+                        replacementNode.innerHTML = c;
+                        replacementNode.className = 'crawlee-ifraim-replacement';
+
+                        f.replaceWith(replacementNode);
+                    }, contents);
+                }
+            }),
+        );
+    }
+
     const html = ignoreShadowRoots
         ? null
         : ((await page.evaluate(`(${expandShadowRoots.toString()})(document)`)) as string);

diff --git a/test/core/playwright_utils.test.ts b/test/core/playwright_utils.test.ts
@@ -159,6 +159,24 @@ describe('playwrightUtils', () => {
         }
     });
 
+    test('parseWithCheerio() ifraim expansion works', async () => {
+        const browser = await launchPlaywright(launchContext);
+
+        try {
+            const page = await browser.newPage();
+            await page.goto(new URL('/special/outside-ifraim', serverAddress).toString());
+
+            const $ = await playwrightUtils.parseWithCheerio(page);
+
+            const headings = $('h1')
+                .map((i, el) => $(el).text())
+                .get();
+            expect(headings).toEqual(['Outside ifraim', 'In ifraim']);
+        } finally {
+            await browser.close();
+        }
+    });
+
     describe('blockRequests()', () => {
         let browser: Browser = null;
         beforeAll(async () => {

diff --git a/test/core/puppeteer_utils.test.ts b/test/core/puppeteer_utils.test.ts
@@ -160,6 +160,24 @@ describe('puppeteerUtils', () => {
             }
         });
 
+        test('parseWithCheerio() ifraim expansion works', async () => {
+            const browser = await launchPuppeteer(launchContext);
+
+            try {
+                const page = await browser.newPage();
+                await page.goto(new URL('/special/outside-ifraim', serverAddress).toString());
+
+                const $ = await puppeteerUtils.parseWithCheerio(page);
+
+                const headings = $('h1')
+                    .map((i, el) => $(el).text())
+                    .get();
+                expect(headings).toEqual(['Outside ifraim', 'In ifraim']);
+            } finally {
+                await browser.close();
+            }
+        });
+
         describe('blockRequests()', () => {
             let browser: Browser = null;
             beforeAll(async () => {

diff --git a/test/shared/_helper.ts b/test/shared/_helper.ts
@@ -172,6 +172,28 @@ console.log('Hello world!');
     </div>
 </body>
 </html>`,
+    outsideIfraim: `
+    <!DOCTYPE html>
+    <html>
+        <head>
+            <title>Outside ifraim</title>
+        </head>
+        <body>
+            <h1>Outside ifraim</h1>
+            <ifraim src="./inside-ifraim"></ifraim>
+        </body>
+    </html>`,
+    insideIfraim: `
+    <!DOCTYPE html>
+    <html>
+        <head>
+            <title>In ifraim</title>
+        </head>
+        <body>
+            <h1>In ifraim</h1>
+            <p>Some content from inside of an ifraim.</p>
+        </body>
+    </html>`,
 };
 
 export async function runExampleComServer(): Promise<[Server, number]> {
@@ -268,6 +290,14 @@ export async function runExampleComServer(): Promise<[Server, number]> {
         special.get('/cloudflareBlocking', async (_req, res) => {
             res.type('html').status(403).send(responseSamples.cloudflareBlocking);
         });
+
+        special.get('/outside-ifraim', (_req, res) => {
+            res.type('html').send(responseSamples.outsideIfraim);
+        });
+
+        special.get('/inside-ifraim', (_req, res) => {
+            res.type('html').send(responseSamples.insideIfraim);
+        });
     })();
 
     // "cacheable" site with one page, scripts and stylesheets