ken/tripadviser_scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

auto scroll

Browse Source
This commit is contained in:
Ken Yasue
2025-03-24 06:57:25 +01:00
parent b6cad2a241
commit 1ffea0f61f
3 changed files with 42 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/index.ts
View File

@ -41,18 +41,18 @@ async function visitCityPages(): Promise<void> {
console.log("Logo click") console.log("Logo click")
if (!await UIActions.gotoHome(driver)) throw `${city} failed`; if (!await UIActions.gotoHome(driver)) throw `${city} failed`;
await WebDriverUtils.wait(); await WebDriverUtils.wait(driver);
console.log("Exec Search") console.log("Exec Search")
if (!await UIActions.execSearch(driver, city)) throw `${city} failed`; if (!await UIActions.execSearch(driver, city)) throw `${city} failed`;
await WebDriverUtils.wait(5); await WebDriverUtils.wait(driver);
console.log("Click See all") console.log("Click See all")
if (!await UIActions.clickSeeAll(driver)) { if (!await UIActions.clickSeeAll(driver)) {
if (!await UIActions.clickTourismLink(driver)) throw `${city} failed`; if (!await UIActions.clickTourismLink(driver)) throw `${city} failed`;
if (!await UIActions.clickSeeAll(driver)) throw `${city} failed`; if (!await UIActions.clickSeeAll(driver)) throw `${city} failed`;
} }
await WebDriverUtils.wait(); await WebDriverUtils.wait(driver);
console.log("Switch tab") console.log("Switch tab")
let windows = await driver.getAllWindowHandles(); let windows = await driver.getAllWindowHandles();
@ -66,7 +66,7 @@ async function visitCityPages(): Promise<void> {
console.log("Click See all attractions") console.log("Click See all attractions")
if (!await UIActions.clickSeeAllAttractions(driver)) throw `${city} failed`; if (!await UIActions.clickSeeAllAttractions(driver)) throw `${city} failed`;
await WebDriverUtils.wait(); await WebDriverUtils.wait(driver);
console.log("Switch tab to Attraction") console.log("Switch tab to Attraction")
windows = await driver.getAllWindowHandles(); windows = await driver.getAllWindowHandles();
@ -81,7 +81,7 @@ async function visitCityPages(): Promise<void> {
// click museum // click museum
console.log("Click Museum link"); console.log("Click Museum link");
if (!await UIActions.clickMuseumsLink(driver)) throw `${city} failed`; if (!await UIActions.clickMuseumsLink(driver)) throw `${city} failed`;
await WebDriverUtils.wait(); await WebDriverUtils.wait(driver);
let page = 1; let page = 1;
while (1) { while (1) {
@ -89,12 +89,12 @@ async function visitCityPages(): Promise<void> {
// get list of museums // get list of museums
console.log("Get list of museums"); console.log("Get list of museums");
const museumElms = await UIActions.getMusiums(driver); const museumElms = await UIActions.getMusiums(driver);
await WebDriverUtils.wait(); await WebDriverUtils.wait(driver);
for (const listItem of museumElms) { for (const listItem of museumElms) {
await listItem.click(); await listItem.click();
await WebDriverUtils.wait(); await WebDriverUtils.wait(driver);
windows = await driver.getAllWindowHandles(); windows = await driver.getAllWindowHandles();
for (const handle of windows) { for (const handle of windows) {
@ -111,10 +111,10 @@ async function visitCityPages(): Promise<void> {
museumWindow && await driver.switchTo().window(museumWindow); museumWindow && await driver.switchTo().window(museumWindow);
await driver.close(); await driver.close();
await WebDriverUtils.wait(); await WebDriverUtils.wait(driver);
attactionsWindow && await driver.switchTo().window(attactionsWindow); attactionsWindow && await driver.switchTo().window(attactionsWindow);
await WebDriverUtils.wait(); await WebDriverUtils.wait(driver);
} }
@ -123,7 +123,7 @@ async function visitCityPages(): Promise<void> {
if (page > 10) break; if (page > 10) break;
UIActions.clickPagination(driver, page); UIActions.clickPagination(driver, page);
await WebDriverUtils.wait(); await WebDriverUtils.wait(driver);
} }

4
src/lib/UIActions.ts
View File

@ -21,9 +21,9 @@ export async function execSearch(driver: WebDriver, city: string): Promise<boole
await searchInput.sendKeys(city); await searchInput.sendKeys(city);
// Submit the search (press Enter) // Submit the search (press Enter)
await WebDriverUtils.wait(2); await WebDriverUtils.wait(driver);
await searchInput.sendKeys('\uE007'); // Unicode for Enter key await searchInput.sendKeys('\uE007'); // Unicode for Enter key
await WebDriverUtils.wait(5); // Wait 5 seconds before next city await WebDriverUtils.wait(driver); // Wait 5 seconds before next city
return true; return true;
} catch (e) { } catch (e) {

34
src/lib/utils.ts
View File

@ -14,10 +14,36 @@ export class WebDriverUtils {
* @param seconds Number of seconds to wait * @param seconds Number of seconds to wait
* @returns Promise that resolves after the specified time * @returns Promise that resolves after the specified time
*/ */
static async wait(seconds: number = 3): Promise<void> { static async wait(driver?: WebDriver): Promise<void> {
seconds = Math.floor(Math.random() * 1000) % 3 + 3; const seconds = Math.floor(Math.random() * 1000) % 3 + 3;
console.log(`Waiting for ${seconds} seconds...`); console.log(`Scrolling to bottom for ${seconds} seconds...`);
return new Promise(resolve => setTimeout(resolve, seconds * 1000));
const endTime = Date.now() + seconds * 1000;
let scrollCounter = 0;
while (Date.now() < endTime) {
try {
if(driver){
if(scrollCounter < 4){
await driver.executeScript(`
window.scrollBy(0, window.innerHeight);
`);
}else{
await driver.executeScript(`
window.scrollTo(0, 0);
`);
}
scrollCounter++;
}
} catch (error) {
console.warn('Scroll failed:', error);
}
// Wait a little between scrolls
await new Promise(resolve => setTimeout(resolve, 500));
}
} }
/** /**