I'm not sure what you've tried or where you're stuck with your attempts exactly, but I can answer generally as this is a common scenario. It's a good illustration that there are rarely one-size-fits-all solutions in web scraping.

If there's a scroll container like this cutting off content, you'll need to "pop" the content out of the container so it scrolls at the top level of the page. I typically use this to do so, removing or hiding all elements except for those in the subtree of the container.

But doing a hard strip here crashes the page, so this script:

• forces the scroll container element tree to be visible
• makes all other elements invisible
• sets general styles to make the visible elements look presentable
• finally, captures the PDF.

const {chromium} = require("playwright"); // ^1.58.0

const url = "https://www.khanacademy.org/math/ap-calculus-ab/ab-diff-analytical-applications-new/ab-5-6b/a/review-analyzing-the-second-derivative-to-find-inflection-points";

let browser;
(async () => {
  browser = await chromium.launch({headless: true});
  const width = 1200;
  const page = await browser.newPage({
    viewport: {width, height: 1200},
  });
  await page.goto(url, {waitUntil: "networkidle"});
  await page
    .locator('[data-testid="content-panel-wrapper"]')
    .evaluate(target => {
      for (let el = target; el; el = el.parentElement) {
        el.style.overflow = "visible";
        el.style.overflowY = "visible";
        el.style.overflowX = "visible";
        el.style.height = "auto";
        el.style.maxHeight = "none";
        el.style.minHeight = "0";
        el.style.position = "static";
      }

      const keep = new Set();
      keep.add(target);
      target.querySelectorAll("*").forEach(el => keep.add(el));

      for (
        let el = target.parentElement;
        el;
        el = el.parentElement
      ) {
        keep.add(el);
      }

      document.querySelectorAll("body *").forEach(el => {
        if (!keep.has(el)) {
          el.style.display = "none";
        }
      });

      target.style.position = "absolute";
      target.style.left = "0";
      target.style.top = "0";
      target.style.width = "100%";
      target.style.maxWidth = "none";
      target.style.margin = "0";
      target.style.padding = "20px";
      document.body.style.margin = "0";
      document.body.style.padding = "0";
      document.body.style.overflow = "visible";
      document.documentElement.style.overflow = "visible";
    });

  await page.waitForTimeout(1000);
  const height = await page.evaluate(() =>
    Math.max(
      document.body.scrollHeight,
      document.documentElement.scrollHeight
    )
  );
  await page.setViewportSize({width, height});
  await page.pdf({
    path: "khan.pdf",
    printBackground: true,
    width: `${width}px`,
    height: `${height}px`,
    margin: {
      top: "0",
      right: "0",
      bottom: "0",
      left: "0",
    },
  });
  console.log("Saved khan.pdf");
})()
  .finally(() => browser?.close());

posted 3 months ago

CC BY-SA 4.0

3mo ago

ggorlen‭

51 reputation 1 2 8 9