Welcome to Software Development on Codidact!
Will you help us build our independent community of developers helping developers? We're small and trying to grow. We welcome questions about all aspects of software development, from design to code to QA and more. Got questions? Got answers? Got code you'd like someone to review? Please join us.
Post History
I'm not sure what you've tried or where you're stuck with your attempts exactly, but I can answer generally as this is a common scenario. It's a good illustration that there are rarely one-size-fit...
#5: Post edited
- I'm not sure what you've tried or where you're stuck with your attempts exactly, but I can answer generally as this is a common scenario. It's a good illustration that there are rarely one-size-fits-all solutions in web scraping.
If there's a scroll container like this cutting off content, you'll need to "pop" the content out of the container. I typically use [this](https://stackoverflow.com/a/71931331/6243352) to do so, removing or hiding all elements except for those in the subtree of the container.- But doing a hard strip here crashes the page, so this script:
- - forces the scroll container element tree to be visible
- - makes all other elements invisible
- - sets general styles to make the visible elements look presentable
- - finally, captures the PDF.
- ```js
- const {chromium} = require("playwright"); // ^1.58.0
- const url = "https://www.khanacademy.org/math/ap-calculus-ab/ab-diff-analytical-applications-new/ab-5-6b/a/review-analyzing-the-second-derivative-to-find-inflection-points";
- let browser;
- (async () => {
- browser = await chromium.launch({headless: true});
- const width = 1200;
- const page = await browser.newPage({
- viewport: {width, height: 1200},
- });
- await page.goto(url, {waitUntil: "networkidle"});
- await page
- .locator('[data-testid="content-panel-wrapper"]')
- .evaluate(target => {
- for (let el = target; el; el = el.parentElement) {
- el.style.overflow = "visible";
- el.style.overflowY = "visible";
- el.style.overflowX = "visible";
- el.style.height = "auto";
- el.style.maxHeight = "none";
- el.style.minHeight = "0";
- el.style.position = "static";
- }
- const keep = new Set();
- keep.add(target);
- target.querySelectorAll("*").forEach(el => keep.add(el));
- for (
- let el = target.parentElement;
- el;
- el = el.parentElement
- ) {
- keep.add(el);
- }
- document.querySelectorAll("body *").forEach(el => {
- if (!keep.has(el)) {
- el.style.display = "none";
- }
- });
- target.style.position = "absolute";
- target.style.left = "0";
- target.style.top = "0";
- target.style.width = "100%";
- target.style.maxWidth = "none";
- target.style.margin = "0";
- target.style.padding = "20px";
- document.body.style.margin = "0";
- document.body.style.padding = "0";
- document.body.style.overflow = "visible";
- document.documentElement.style.overflow = "visible";
- });
- await page.waitForTimeout(1000);
- const height = await page.evaluate(() =>
- Math.max(
- document.body.scrollHeight,
- document.documentElement.scrollHeight
- )
- );
- await page.setViewportSize({width, height});
- await page.pdf({
- path: "khan.pdf",
- printBackground: true,
- width: `${width}px`,
- height: `${height}px`,
- margin: {
- top: "0",
- right: "0",
- bottom: "0",
- left: "0",
- },
- });
- console.log("Saved khan.pdf");
- })()
- .finally(() => browser?.close());
- ```
- I'm not sure what you've tried or where you're stuck with your attempts exactly, but I can answer generally as this is a common scenario. It's a good illustration that there are rarely one-size-fits-all solutions in web scraping.
- If there's a scroll container like this cutting off content, you'll need to "pop" the content out of the container so it scrolls at the top level of the page. I typically use [this](https://stackoverflow.com/a/71931331/6243352) to do so, removing or hiding all elements except for those in the subtree of the container.
- But doing a hard strip here crashes the page, so this script:
- - forces the scroll container element tree to be visible
- - makes all other elements invisible
- - sets general styles to make the visible elements look presentable
- - finally, captures the PDF.
- ```js
- const {chromium} = require("playwright"); // ^1.58.0
- const url = "https://www.khanacademy.org/math/ap-calculus-ab/ab-diff-analytical-applications-new/ab-5-6b/a/review-analyzing-the-second-derivative-to-find-inflection-points";
- let browser;
- (async () => {
- browser = await chromium.launch({headless: true});
- const width = 1200;
- const page = await browser.newPage({
- viewport: {width, height: 1200},
- });
- await page.goto(url, {waitUntil: "networkidle"});
- await page
- .locator('[data-testid="content-panel-wrapper"]')
- .evaluate(target => {
- for (let el = target; el; el = el.parentElement) {
- el.style.overflow = "visible";
- el.style.overflowY = "visible";
- el.style.overflowX = "visible";
- el.style.height = "auto";
- el.style.maxHeight = "none";
- el.style.minHeight = "0";
- el.style.position = "static";
- }
- const keep = new Set();
- keep.add(target);
- target.querySelectorAll("*").forEach(el => keep.add(el));
- for (
- let el = target.parentElement;
- el;
- el = el.parentElement
- ) {
- keep.add(el);
- }
- document.querySelectorAll("body *").forEach(el => {
- if (!keep.has(el)) {
- el.style.display = "none";
- }
- });
- target.style.position = "absolute";
- target.style.left = "0";
- target.style.top = "0";
- target.style.width = "100%";
- target.style.maxWidth = "none";
- target.style.margin = "0";
- target.style.padding = "20px";
- document.body.style.margin = "0";
- document.body.style.padding = "0";
- document.body.style.overflow = "visible";
- document.documentElement.style.overflow = "visible";
- });
- await page.waitForTimeout(1000);
- const height = await page.evaluate(() =>
- Math.max(
- document.body.scrollHeight,
- document.documentElement.scrollHeight
- )
- );
- await page.setViewportSize({width, height});
- await page.pdf({
- path: "khan.pdf",
- printBackground: true,
- width: `${width}px`,
- height: `${height}px`,
- margin: {
- top: "0",
- right: "0",
- bottom: "0",
- left: "0",
- },
- });
- console.log("Saved khan.pdf");
- })()
- .finally(() => browser?.close());
- ```
#4: Post edited
I'm not sure what you've tried or where you're stuck with your attempts exactly, but I can answer generally.- If there's a scroll container like this cutting off content, you'll need to "pop" the content out of the container. I typically use [this](https://stackoverflow.com/a/71931331/6243352) to do so, removing or hiding all elements except for those in the subtree of the container.
- But doing a hard strip here crashes the page, so this script:
- - forces the scroll container element tree to be visible
- - makes all other elements invisible
- - sets general styles to make the visible elements look presentable
- - finally, captures the PDF.
- ```js
- const {chromium} = require("playwright"); // ^1.58.0
- const url = "https://www.khanacademy.org/math/ap-calculus-ab/ab-diff-analytical-applications-new/ab-5-6b/a/review-analyzing-the-second-derivative-to-find-inflection-points";
- let browser;
- (async () => {
- browser = await chromium.launch({headless: true});
- const width = 1200;
- const page = await browser.newPage({
- viewport: {width, height: 1200},
- });
- await page.goto(url, {waitUntil: "networkidle"});
- await page
- .locator('[data-testid="content-panel-wrapper"]')
- .evaluate(target => {
- for (let el = target; el; el = el.parentElement) {
- el.style.overflow = "visible";
- el.style.overflowY = "visible";
- el.style.overflowX = "visible";
- el.style.height = "auto";
- el.style.maxHeight = "none";
- el.style.minHeight = "0";
- el.style.position = "static";
- }
- const keep = new Set();
- keep.add(target);
- target.querySelectorAll("*").forEach(el => keep.add(el));
- for (
- let el = target.parentElement;
- el;
- el = el.parentElement
- ) {
- keep.add(el);
- }
- document.querySelectorAll("body *").forEach(el => {
- if (!keep.has(el)) {
- el.style.display = "none";
- }
- });
- target.style.position = "absolute";
- target.style.left = "0";
- target.style.top = "0";
- target.style.width = "100%";
- target.style.maxWidth = "none";
- target.style.margin = "0";
- target.style.padding = "20px";
- document.body.style.margin = "0";
- document.body.style.padding = "0";
- document.body.style.overflow = "visible";
- document.documentElement.style.overflow = "visible";
- });
- await page.waitForTimeout(1000);
- const height = await page.evaluate(() =>
- Math.max(
- document.body.scrollHeight,
- document.documentElement.scrollHeight
- )
- );
- await page.setViewportSize({width, height});
- await page.pdf({
- path: "khan.pdf",
- printBackground: true,
- width: `${width}px`,
- height: `${height}px`,
- margin: {
- top: "0",
- right: "0",
- bottom: "0",
- left: "0",
- },
- });
- console.log("Saved khan.pdf");
- })()
- .finally(() => browser?.close());
- ```
- I'm not sure what you've tried or where you're stuck with your attempts exactly, but I can answer generally as this is a common scenario. It's a good illustration that there are rarely one-size-fits-all solutions in web scraping.
- If there's a scroll container like this cutting off content, you'll need to "pop" the content out of the container. I typically use [this](https://stackoverflow.com/a/71931331/6243352) to do so, removing or hiding all elements except for those in the subtree of the container.
- But doing a hard strip here crashes the page, so this script:
- - forces the scroll container element tree to be visible
- - makes all other elements invisible
- - sets general styles to make the visible elements look presentable
- - finally, captures the PDF.
- ```js
- const {chromium} = require("playwright"); // ^1.58.0
- const url = "https://www.khanacademy.org/math/ap-calculus-ab/ab-diff-analytical-applications-new/ab-5-6b/a/review-analyzing-the-second-derivative-to-find-inflection-points";
- let browser;
- (async () => {
- browser = await chromium.launch({headless: true});
- const width = 1200;
- const page = await browser.newPage({
- viewport: {width, height: 1200},
- });
- await page.goto(url, {waitUntil: "networkidle"});
- await page
- .locator('[data-testid="content-panel-wrapper"]')
- .evaluate(target => {
- for (let el = target; el; el = el.parentElement) {
- el.style.overflow = "visible";
- el.style.overflowY = "visible";
- el.style.overflowX = "visible";
- el.style.height = "auto";
- el.style.maxHeight = "none";
- el.style.minHeight = "0";
- el.style.position = "static";
- }
- const keep = new Set();
- keep.add(target);
- target.querySelectorAll("*").forEach(el => keep.add(el));
- for (
- let el = target.parentElement;
- el;
- el = el.parentElement
- ) {
- keep.add(el);
- }
- document.querySelectorAll("body *").forEach(el => {
- if (!keep.has(el)) {
- el.style.display = "none";
- }
- });
- target.style.position = "absolute";
- target.style.left = "0";
- target.style.top = "0";
- target.style.width = "100%";
- target.style.maxWidth = "none";
- target.style.margin = "0";
- target.style.padding = "20px";
- document.body.style.margin = "0";
- document.body.style.padding = "0";
- document.body.style.overflow = "visible";
- document.documentElement.style.overflow = "visible";
- });
- await page.waitForTimeout(1000);
- const height = await page.evaluate(() =>
- Math.max(
- document.body.scrollHeight,
- document.documentElement.scrollHeight
- )
- );
- await page.setViewportSize({width, height});
- await page.pdf({
- path: "khan.pdf",
- printBackground: true,
- width: `${width}px`,
- height: `${height}px`,
- margin: {
- top: "0",
- right: "0",
- bottom: "0",
- left: "0",
- },
- });
- console.log("Saved khan.pdf");
- })()
- .finally(() => browser?.close());
- ```
#3: Post edited
- I'm not sure what you've tried or where you're stuck with your attempts exactly, but I can answer generally.
- If there's a scroll container like this cutting off content, you'll need to "pop" the content out of the container. I typically use [this](https://stackoverflow.com/a/71931331/6243352) to do so, removing or hiding all elements except for those in the subtree of the container.
- But doing a hard strip here crashes the page, so this script:
- forces the scroll container element tree visible- - makes all other elements invisible
- sets some general styles to make the visible elements look presentable- - finally, captures the PDF.
- ```js
- const {chromium} = require("playwright"); // ^1.58.0
- const url = "https://www.khanacademy.org/math/ap-calculus-ab/ab-diff-analytical-applications-new/ab-5-6b/a/review-analyzing-the-second-derivative-to-find-inflection-points";
- let browser;
- (async () => {
- browser = await chromium.launch({headless: true});
- const width = 1200;
- const page = await browser.newPage({
- viewport: {width, height: 1200},
- });
- await page.goto(url, {waitUntil: "networkidle"});
- await page
- .locator('[data-testid="content-panel-wrapper"]')
- .evaluate(target => {
- for (let el = target; el; el = el.parentElement) {
- el.style.overflow = "visible";
- el.style.overflowY = "visible";
- el.style.overflowX = "visible";
- el.style.height = "auto";
- el.style.maxHeight = "none";
- el.style.minHeight = "0";
- el.style.position = "static";
- }
- const keep = new Set();
- keep.add(target);
- target.querySelectorAll("*").forEach(el => keep.add(el));
- for (
- let el = target.parentElement;
- el;
- el = el.parentElement
- ) {
- keep.add(el);
- }
- document.querySelectorAll("body *").forEach(el => {
- if (!keep.has(el)) {
- el.style.display = "none";
- }
- });
- target.style.position = "absolute";
- target.style.left = "0";
- target.style.top = "0";
- target.style.width = "100%";
- target.style.maxWidth = "none";
- target.style.margin = "0";
- target.style.padding = "20px";
- document.body.style.margin = "0";
- document.body.style.padding = "0";
- document.body.style.overflow = "visible";
- document.documentElement.style.overflow = "visible";
- });
- await page.waitForTimeout(1000);
- const height = await page.evaluate(() =>
- Math.max(
- document.body.scrollHeight,
- document.documentElement.scrollHeight
- )
- );
- await page.setViewportSize({width, height});
- await page.pdf({
- path: "khan.pdf",
- printBackground: true,
- width: `${width}px`,
- height: `${height}px`,
- margin: {
- top: "0",
- right: "0",
- bottom: "0",
- left: "0",
- },
- });
- console.log("Saved khan.pdf");
- })()
- .finally(() => browser?.close());
- ```
- I'm not sure what you've tried or where you're stuck with your attempts exactly, but I can answer generally.
- If there's a scroll container like this cutting off content, you'll need to "pop" the content out of the container. I typically use [this](https://stackoverflow.com/a/71931331/6243352) to do so, removing or hiding all elements except for those in the subtree of the container.
- But doing a hard strip here crashes the page, so this script:
- - forces the scroll container element tree to be visible
- - makes all other elements invisible
- - sets general styles to make the visible elements look presentable
- - finally, captures the PDF.
- ```js
- const {chromium} = require("playwright"); // ^1.58.0
- const url = "https://www.khanacademy.org/math/ap-calculus-ab/ab-diff-analytical-applications-new/ab-5-6b/a/review-analyzing-the-second-derivative-to-find-inflection-points";
- let browser;
- (async () => {
- browser = await chromium.launch({headless: true});
- const width = 1200;
- const page = await browser.newPage({
- viewport: {width, height: 1200},
- });
- await page.goto(url, {waitUntil: "networkidle"});
- await page
- .locator('[data-testid="content-panel-wrapper"]')
- .evaluate(target => {
- for (let el = target; el; el = el.parentElement) {
- el.style.overflow = "visible";
- el.style.overflowY = "visible";
- el.style.overflowX = "visible";
- el.style.height = "auto";
- el.style.maxHeight = "none";
- el.style.minHeight = "0";
- el.style.position = "static";
- }
- const keep = new Set();
- keep.add(target);
- target.querySelectorAll("*").forEach(el => keep.add(el));
- for (
- let el = target.parentElement;
- el;
- el = el.parentElement
- ) {
- keep.add(el);
- }
- document.querySelectorAll("body *").forEach(el => {
- if (!keep.has(el)) {
- el.style.display = "none";
- }
- });
- target.style.position = "absolute";
- target.style.left = "0";
- target.style.top = "0";
- target.style.width = "100%";
- target.style.maxWidth = "none";
- target.style.margin = "0";
- target.style.padding = "20px";
- document.body.style.margin = "0";
- document.body.style.padding = "0";
- document.body.style.overflow = "visible";
- document.documentElement.style.overflow = "visible";
- });
- await page.waitForTimeout(1000);
- const height = await page.evaluate(() =>
- Math.max(
- document.body.scrollHeight,
- document.documentElement.scrollHeight
- )
- );
- await page.setViewportSize({width, height});
- await page.pdf({
- path: "khan.pdf",
- printBackground: true,
- width: `${width}px`,
- height: `${height}px`,
- margin: {
- top: "0",
- right: "0",
- bottom: "0",
- left: "0",
- },
- });
- console.log("Saved khan.pdf");
- })()
- .finally(() => browser?.close());
- ```
#2: Post edited
- I'm not sure what you've tried or where you're stuck with your attempts exactly, but I can answer generally.
- If there's a scroll container like this cutting off content, you'll need to "pop" the content out of the container. I typically use [this](https://stackoverflow.com/a/71931331/6243352) to do so, removing or hiding all elements except for those in the subtree of the container.
- But doing a hard strip here crashes the page, so this script:
- - forces the scroll container element tree visible
- - makes all other elements invisible
- sets some general styles to make the visible elements look presentable.- ```js
- const {chromium} = require("playwright"); // ^1.58.0
- const url = "https://www.khanacademy.org/math/ap-calculus-ab/ab-diff-analytical-applications-new/ab-5-6b/a/review-analyzing-the-second-derivative-to-find-inflection-points";
- let browser;
- (async () => {
- browser = await chromium.launch({headless: true});
- const width = 1200;
- const page = await browser.newPage({
- viewport: {width, height: 1200},
- });
- await page.goto(url, {waitUntil: "networkidle"});
- await page
- .locator('[data-testid="content-panel-wrapper"]')
- .evaluate(target => {
- for (let el = target; el; el = el.parentElement) {
- el.style.overflow = "visible";
- el.style.overflowY = "visible";
- el.style.overflowX = "visible";
- el.style.height = "auto";
- el.style.maxHeight = "none";
- el.style.minHeight = "0";
- el.style.position = "static";
- }
- const keep = new Set();
- keep.add(target);
- target.querySelectorAll("*").forEach(el => keep.add(el));
- for (
- let el = target.parentElement;
- el;
- el = el.parentElement
- ) {
- keep.add(el);
- }
- document.querySelectorAll("body *").forEach(el => {
- if (!keep.has(el)) {
- el.style.display = "none";
- }
- });
- target.style.position = "absolute";
- target.style.left = "0";
- target.style.top = "0";
- target.style.width = "100%";
- target.style.maxWidth = "none";
- target.style.margin = "0";
- target.style.padding = "20px";
- document.body.style.margin = "0";
- document.body.style.padding = "0";
- document.body.style.overflow = "visible";
- document.documentElement.style.overflow = "visible";
- });
- await page.waitForTimeout(1000);
- const height = await page.evaluate(() =>
- Math.max(
- document.body.scrollHeight,
- document.documentElement.scrollHeight
- )
- );
- await page.setViewportSize({width, height});
- await page.pdf({
- path: "khan.pdf",
- printBackground: true,
- width: `${width}px`,
- height: `${height}px`,
- margin: {
- top: "0",
- right: "0",
- bottom: "0",
- left: "0",
- },
- });
- console.log("Saved khan.pdf");
- })()
- .finally(() => browser?.close());
- ```
- I'm not sure what you've tried or where you're stuck with your attempts exactly, but I can answer generally.
- If there's a scroll container like this cutting off content, you'll need to "pop" the content out of the container. I typically use [this](https://stackoverflow.com/a/71931331/6243352) to do so, removing or hiding all elements except for those in the subtree of the container.
- But doing a hard strip here crashes the page, so this script:
- - forces the scroll container element tree visible
- - makes all other elements invisible
- - sets some general styles to make the visible elements look presentable
- - finally, captures the PDF.
- ```js
- const {chromium} = require("playwright"); // ^1.58.0
- const url = "https://www.khanacademy.org/math/ap-calculus-ab/ab-diff-analytical-applications-new/ab-5-6b/a/review-analyzing-the-second-derivative-to-find-inflection-points";
- let browser;
- (async () => {
- browser = await chromium.launch({headless: true});
- const width = 1200;
- const page = await browser.newPage({
- viewport: {width, height: 1200},
- });
- await page.goto(url, {waitUntil: "networkidle"});
- await page
- .locator('[data-testid="content-panel-wrapper"]')
- .evaluate(target => {
- for (let el = target; el; el = el.parentElement) {
- el.style.overflow = "visible";
- el.style.overflowY = "visible";
- el.style.overflowX = "visible";
- el.style.height = "auto";
- el.style.maxHeight = "none";
- el.style.minHeight = "0";
- el.style.position = "static";
- }
- const keep = new Set();
- keep.add(target);
- target.querySelectorAll("*").forEach(el => keep.add(el));
- for (
- let el = target.parentElement;
- el;
- el = el.parentElement
- ) {
- keep.add(el);
- }
- document.querySelectorAll("body *").forEach(el => {
- if (!keep.has(el)) {
- el.style.display = "none";
- }
- });
- target.style.position = "absolute";
- target.style.left = "0";
- target.style.top = "0";
- target.style.width = "100%";
- target.style.maxWidth = "none";
- target.style.margin = "0";
- target.style.padding = "20px";
- document.body.style.margin = "0";
- document.body.style.padding = "0";
- document.body.style.overflow = "visible";
- document.documentElement.style.overflow = "visible";
- });
- await page.waitForTimeout(1000);
- const height = await page.evaluate(() =>
- Math.max(
- document.body.scrollHeight,
- document.documentElement.scrollHeight
- )
- );
- await page.setViewportSize({width, height});
- await page.pdf({
- path: "khan.pdf",
- printBackground: true,
- width: `${width}px`,
- height: `${height}px`,
- margin: {
- top: "0",
- right: "0",
- bottom: "0",
- left: "0",
- },
- });
- console.log("Saved khan.pdf");
- })()
- .finally(() => browser?.close());
- ```
#1: Initial revision
I'm not sure what you've tried or where you're stuck with your attempts exactly, but I can answer generally.
If there's a scroll container like this cutting off content, you'll need to "pop" the content out of the container. I typically use [this](https://stackoverflow.com/a/71931331/6243352) to do so, removing or hiding all elements except for those in the subtree of the container.
But doing a hard strip here crashes the page, so this script:
- forces the scroll container element tree visible
- makes all other elements invisible
- sets some general styles to make the visible elements look presentable.
```js
const {chromium} = require("playwright"); // ^1.58.0
const url = "https://www.khanacademy.org/math/ap-calculus-ab/ab-diff-analytical-applications-new/ab-5-6b/a/review-analyzing-the-second-derivative-to-find-inflection-points";
let browser;
(async () => {
browser = await chromium.launch({headless: true});
const width = 1200;
const page = await browser.newPage({
viewport: {width, height: 1200},
});
await page.goto(url, {waitUntil: "networkidle"});
await page
.locator('[data-testid="content-panel-wrapper"]')
.evaluate(target => {
for (let el = target; el; el = el.parentElement) {
el.style.overflow = "visible";
el.style.overflowY = "visible";
el.style.overflowX = "visible";
el.style.height = "auto";
el.style.maxHeight = "none";
el.style.minHeight = "0";
el.style.position = "static";
}
const keep = new Set();
keep.add(target);
target.querySelectorAll("*").forEach(el => keep.add(el));
for (
let el = target.parentElement;
el;
el = el.parentElement
) {
keep.add(el);
}
document.querySelectorAll("body *").forEach(el => {
if (!keep.has(el)) {
el.style.display = "none";
}
});
target.style.position = "absolute";
target.style.left = "0";
target.style.top = "0";
target.style.width = "100%";
target.style.maxWidth = "none";
target.style.margin = "0";
target.style.padding = "20px";
document.body.style.margin = "0";
document.body.style.padding = "0";
document.body.style.overflow = "visible";
document.documentElement.style.overflow = "visible";
});
await page.waitForTimeout(1000);
const height = await page.evaluate(() =>
Math.max(
document.body.scrollHeight,
document.documentElement.scrollHeight
)
);
await page.setViewportSize({width, height});
await page.pdf({
path: "khan.pdf",
printBackground: true,
width: `${width}px`,
height: `${height}px`,
margin: {
top: "0",
right: "0",
bottom: "0",
left: "0",
},
});
console.log("Saved khan.pdf");
})()
.finally(() => browser?.close());
```
