遍历链接数组会导致导航超时错误-puppeteer

如何解决遍历链接数组会导致导航超时错误-puppeteer

我有一个按钮元素数组，我想要一个按钮一次单击，并对每个打开的新选项卡执行此操作：

抓取一些信息并存储在名为“ providers”的数组中
关闭该标签页

虽然我能够执行此操作，但由于在browser.pages（）之前使用了导航组件，所以我一直收到超时错误。如果删除组件，则会收到另一个超时错误。另外，每次我运行该程序时，在按钮数组进行不同次数的迭代后，都会遇到超时错误。这是我的代码：

const puppeteer = require("puppeteer");

(async () => {
  try {
    const browser = await puppeteer.launch({
      headless: false,});
    const page = await browser.newPage();

    //google.com
    await page.setExtraHTTPHeaders({ "Accept-Language": "en-US" });
    await page.goto("https://google.com");
    await page.type("input.gLFyf.gsfi","hotels in london");
    await page.keyboard.press("Enter");

    //search results
    await page.waitForXPath('//span[contains(text(),"View ")]');
    const btn1 = await page.$x('//span[contains(text(),"View ")]');
    await btn1[0].click();

    //list of hotels
    await page.waitForXPath('//span[contains(text(),"Learn more")]');

    let hotels = [];
    
    //buttons array that contains a list of buttons
    let buttons = await page.$x("//button[contains(.,'View prices')]");
 
    //prints a different value each time the program is run
    console.log(buttons.length);
 
    //looping through buttons array
    for (var i = 0; i < buttons.length; i++) {

      //i = 1 or 0 when program hangs 
      console.log("got here " + I);

      //*******************************cause of timeout error******************************************

      await page.setDefaultNavigationTimeout(0);
      await Promise.all([
        page.waitForNavigation({ waitUntil: "load",timeout: 0 }),buttons[i].click(),]);

      //***********************************************************************************************

      //getting all open tabs in an array
      const pages = await browser.pages();
      const page2 = pages[pages.length - 1];
      console.log(pages.length);

      //newly opened tab,sometimes program hangs before opening a new tab
      await page2
        .waitForSelector(
          "#prices > c-wiz > div > div.G86l0b > div > div > div > div > div > section > div.Hkwcrd.q9W60.A5WLXb.fLClSe > c-wiz > div > div > span > div > div > div > div > div > a > div > div.cFdfnb > div > span.mK0tQb > span",{ timeout: 30000 }
        )
        .catch(() => console.log("Class doesn't exist!"));

      /*-----------------scraping information on new tab ----------------------------------*/

      console.log("going to start collecting providers");
      let providers = await page2.evaluate(() => {
        let data = [];
        let elements = document.querySelectorAll(
          "#prices > c-wiz > div > div.G86l0b > div > div > div > div > div > section > div.Hkwcrd.q9W60.A5WLXb.fLClSe > c-wiz > div > div > span > div > div > div > div > div > a > div > div.cFdfnb > div > span.mK0tQb > span"
        );
        for (var element of elements) data.push(element.textContent);
        return data;
      });
      console.log(providers.length);
      console.log("all done");
      console.log(providers);
      hotels.push(providers);

      //closing the new tab
      page2.close();
    }
    
    await browser.close();
    return hotels;
  } catch (err) {
    console.error(err);
  }
})()
  .then((resolvedValue) => {
    console.log(resolvedValue);
  })
  .catch((rejectedValue) => {
    console.log(rejectedValue);
  });

要消除该错误，我使用了超时：0和setDefaultNavigationTimeout（0），但是现在程序冻结了。这是我在禁用超时获取之前遇到的错误：

TimeoutError: Navigation timeout of 30000 ms exceeded
    at C:\Users\Me\Desktop\web_scraping_practice\node_modules\puppeteer\lib\LifecycleWatcher.js:100:111
    at async FrameManager.waitForFrameNavigation (C:\Users\Me\Desktop\web_scraping_practice\node_modules\puppeteer\lib\FrameManager.js:107:23)
    at async Frame.waitForNavigation (C:\Users\Me\Desktop\web_scraping_practice\node_modules\puppeteer\lib\FrameManager.js:298:16)
    at async Page.waitForNavigation (C:\Users\Me\Desktop\web_scraping_practice\node_modules\puppeteer\lib\Page.js:560:16)
    at async Promise.all (index 0)
    at async C:\Users\Me\Desktop\web_scraping_practice\backend.js:41:7
  -- ASYNC --
    at Frame.<anonymous> (C:\Users\Me\Desktop\web_scraping_practice\node_modules\puppeteer\lib\helper.js:116:19)
    at Page.waitForNavigation (C:\Users\Me\Desktop\web_scraping_practice\node_modules\puppeteer\lib\Page.js:560:53)
    at Page.<anonymous> (C:\Users\Me\Desktop\web_scraping_practice\node_modules\puppeteer\lib\helper.js:117:27)
    at C:\Users\Me\Desktop\web_scraping_practice\backend.js:42:14
    at processTicksAndRejections (internal/process/task_queues.js:97:5) {
  name: 'TimeoutError'
}
undefined

谢谢

解决方法

尝试运行您的代码，如果您按其内容搜索跨度，似乎最好对Chromium语言环境进行硬编码，因为在我的浏览器中，它们不是英语的。但是我做了一些调整，设法打开了一个包含酒店详细信息的标签。问题是这个选择器：

$("#prices > c-wiz > div > div.G86l0b > div > div > div > div > div > section > div.Hkwcrd.q9W60.A5WLXb.fLClSe > c-wiz > div > div > span > div > div > div > div > div > a > div > div.cFdfnb > div > span.mK0tQb > span");

不幸的是，这个东西渲染了null。我相信这组类div.Hkwcrd.q9W60.A5WLXb.fLClSe是动态生成的。不知道您实际上要提取什么信息，但是我会尝试通过此data-click-type属性来查找DOM元素。就我而言，它产生：

document.querySelectorAll("div[data-click-type='283']");
NodeList(18) [div.YPrvOd,div.YPrvOd,div.YPrvOd]

似乎是房间类型（高级双人间等）。 “ 268”点击类型似乎是包含酒店（预订，hotels.com等）的网站

下面的代码：

const puppeteer = require("puppeteer");

(async () => {
  try {
    const browser = await puppeteer.launch({
      headless: false,});
    const page = await browser.newPage();

    //google.com
    await page.setExtraHTTPHeaders({ "Accept-Language": "en-US" });
    await page.goto("https://google.com");
    await page.type("input.gLFyf.gsfi","hotels in london");
    await page.keyboard.press("Enter");

    //search results
    await page.waitForXPath('//span[contains(text(),"View ")]');
    const btn1 = await page.$x('//span[contains(text(),"View ")]');
    await btn1[0].click();

    //list of hotels
    await page.waitForXPath('//span[contains(text(),"Learn more")]');

    let hotels = [];

    //buttons array that contains a list of buttons
    let buttons = await page.$x("//button[contains(.,'View prices')]");

    //prints a different value each time the program is run
    console.log(buttons.length);

    //looping through buttons array
    for (var i = 0; i < buttons.length; i++) {

      //i = 1 or 0 when program hangs
      console.log("got here " + i);

      //*******************************cause of timeout error******************************************

      await page.setDefaultNavigationTimeout(0);
      await Promise.all([
        page.waitForNavigation({ waitUntil: "load",timeout: 0 }),buttons[i].click(),]);

      //***********************************************************************************************

      //getting all open tabs in an array
      const pages = await browser.pages();
      const page2 = pages[pages.length - 1];
      console.log(pages.length);

      //newly opened tab,sometimes program hangs before opening a new tab
      await page2
        .waitForSelector(
          "span[data-click-type='268']",{ timeout: 30000 }
        )
        .catch(() => console.log("Class doesn't exist!"));

      /*-----------------scraping information on new tab ----------------------------------*/

      console.log("going to start collecting providers");
      let providers = await page2.evaluate(() => {
        let data = [];
        let elements = document.querySelectorAll(
          "span[data-click-type='268']"
        );
        for (var element of elements) data.push(element.textContent);
        return data;
      });
      console.log(providers.length);
      console.log("all done");
      console.log(providers);
      hotels.push(providers);

      //closing the new tab
      page2.close();
    }

    await browser.close();
    return hotels;
  } catch (err) {
    console.error(err);
  }
})()
  .then((resolvedValue) => {
    console.log(resolvedValue);
  })
  .catch((rejectedValue) => {
    console.log(rejectedValue);
  });

在我的情况下，呈现以下内容：

(node:16816) ExperimentalWarning: The fs.promises API is experimental
12
got here 0
3
going to start collecting providers
16
all done
[ 'Booking.com','Tripadvisor.com','Agoda','Hotels.com','Booking.com','Expedia.com','Destinia','Stayforlong.com','Trip.com','ebookers.ie','Etrip','ZenHotels.com','Nustay.com' ]
got here 1

我相信，这是providers的列表。通知使用的选择器：span[data-click-type='268']。

遍历链接数组会导致导航超时错误-puppeteer

如何解决遍历链接数组会导致导航超时错误-puppeteer

解决方法

相关推荐