ken/tripadviser_scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Ken Yasue
2025-03-24 06:47:38 +01:00
parent d0bfe15fa4
commit 6f36809932
2 changed files with 13 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/index.ts
View File

@ -50,7 +50,7 @@ async function visitCityPages(): Promise<void> {
console.log("Logo click") console.log("Logo click")
if (!await UIActions.gotoHome(driver)) throw `${city} failed`; if (!await UIActions.gotoHome(driver)) throw `${city} failed`;
await WebDriverUtils.wait(5); await WebDriverUtils.wait();
console.log("Exec Search") console.log("Exec Search")
if (!await UIActions.execSearch(driver, city)) throw `${city} failed`; if (!await UIActions.execSearch(driver, city)) throw `${city} failed`;
@ -61,7 +61,7 @@ async function visitCityPages(): Promise<void> {
if (!await UIActions.clickTourismLink(driver)) throw `${city} failed`; if (!await UIActions.clickTourismLink(driver)) throw `${city} failed`;
if (!await UIActions.clickSeeAll(driver)) throw `${city} failed`; if (!await UIActions.clickSeeAll(driver)) throw `${city} failed`;
} }
await WebDriverUtils.wait(5); await WebDriverUtils.wait();
console.log("Switch tab") console.log("Switch tab")
let windows = await driver.getAllWindowHandles(); let windows = await driver.getAllWindowHandles();
@ -75,7 +75,7 @@ async function visitCityPages(): Promise<void> {
console.log("Click See all attractions") console.log("Click See all attractions")
if (!await UIActions.clickSeeAllAttractions(driver)) throw `${city} failed`; if (!await UIActions.clickSeeAllAttractions(driver)) throw `${city} failed`;
await WebDriverUtils.wait(5); await WebDriverUtils.wait();
console.log("Switch tab to Attraction") console.log("Switch tab to Attraction")
windows = await driver.getAllWindowHandles(); windows = await driver.getAllWindowHandles();
@ -90,7 +90,7 @@ async function visitCityPages(): Promise<void> {
// click museum // click museum
console.log("Click Museum link"); console.log("Click Museum link");
if (!await UIActions.clickMuseumsLink(driver)) throw `${city} failed`; if (!await UIActions.clickMuseumsLink(driver)) throw `${city} failed`;
await WebDriverUtils.wait(5); await WebDriverUtils.wait();
let page = 1; let page = 1;
while (1) { while (1) {
@ -98,12 +98,12 @@ async function visitCityPages(): Promise<void> {
// get list of museums // get list of museums
console.log("Get list of museums"); console.log("Get list of museums");
const museumElms = await UIActions.getMusiums(driver); const museumElms = await UIActions.getMusiums(driver);
await WebDriverUtils.wait(1); await WebDriverUtils.wait();
for (const listItem of museumElms) { for (const listItem of museumElms) {
await listItem.click(); await listItem.click();
await WebDriverUtils.wait(3); await WebDriverUtils.wait();
windows = await driver.getAllWindowHandles(); windows = await driver.getAllWindowHandles();
for (const handle of windows) { for (const handle of windows) {
@ -120,10 +120,10 @@ async function visitCityPages(): Promise<void> {
museumWindow && await driver.switchTo().window(museumWindow); museumWindow && await driver.switchTo().window(museumWindow);
await driver.close(); await driver.close();
await WebDriverUtils.wait(1); await WebDriverUtils.wait();
attactionsWindow && await driver.switchTo().window(attactionsWindow); attactionsWindow && await driver.switchTo().window(attactionsWindow);
await WebDriverUtils.wait(1); await WebDriverUtils.wait();
} }
@ -132,7 +132,7 @@ async function visitCityPages(): Promise<void> {
if (page > 10) break; if (page > 10) break;
UIActions.clickPagination(driver, page); UIActions.clickPagination(driver, page);
await WebDriverUtils.wait(5); await WebDriverUtils.wait();
} }
@ -142,7 +142,7 @@ async function visitCityPages(): Promise<void> {
if (i < cities.length - 1) { if (i < cities.length - 1) {
console.log(`Waiting for 5000 seconds before next city...`); console.log(`Waiting for 5000 seconds before next city...`);
await WebDriverUtils.wait(5); // Wait 5000 seconds before next city await WebDriverUtils.wait(); // Wait 5000 seconds before next city
} }
} catch (error) { } catch (error) {
@ -150,7 +150,7 @@ async function visitCityPages(): Promise<void> {
// If the button is not found within the timeout, log and continue to the next city // If the button is not found within the timeout, log and continue to the next city
console.log(`No Museums button found for ${city}. Moving to next city after 5 seconds...`); console.log(`No Museums button found for ${city}. Moving to next city after 5 seconds...`);
await WebDriverUtils.wait(5); // Wait 5 seconds before next city await WebDriverUtils.wait(); // Wait 5 seconds before next city
} }
} }

3
src/lib/utils.ts
View File

@ -13,7 +13,8 @@ export class WebDriverUtils {
* @param seconds Number of seconds to wait * @param seconds Number of seconds to wait
* @returns Promise that resolves after the specified time * @returns Promise that resolves after the specified time
*/ */
static async wait(seconds: number): Promise<void> { static async wait(seconds: number = 3): Promise<void> {
seconds = Math.floor(Math.random() * 1000) % 3 + 3;
console.log(`Waiting for ${seconds} seconds...`); console.log(`Waiting for ${seconds} seconds...`);
return new Promise(resolve => setTimeout(resolve, seconds * 1000)); return new Promise(resolve => setTimeout(resolve, seconds * 1000));
} }