ken/tripadviser_scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

save work

Browse Source
This commit is contained in:
Ken Yasue
2025-03-24 19:00:26 +01:00
parent fe177abd85
commit 47134525b8
3 changed files with 13 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/index.ts
View File

@ -23,7 +23,7 @@ async function visitCityPages(): Promise<void> {
console.log('Connecting to existing Chrome browser...'); console.log('Connecting to existing Chrome browser...');
const driver = await useChrome(); const driver = await useExistingChrome();
if (!driver) return; if (!driver) return;
// Visit each city's TripAdvisor page // Visit each city's TripAdvisor page
@ -48,14 +48,10 @@ async function visitCityPages(): Promise<void> {
await WebDriverUtils.wait(driver); await WebDriverUtils.wait(driver);
console.log("Click See all") console.log("Click See all")
let seeAllUrl = await UIActions.getSeeAllUrl(driver); if (!await UIActions.clickSeeAllAttractions(driver)) {
if (seeAllUrl.length == 0) {
if (!await UIActions.clickTourismLink(driver)) throw `${city} failed`; if (!await UIActions.clickTourismLink(driver)) throw `${city} failed`;
seeAllUrl = await UIActions.getSeeAllUrl(driver); if (!await UIActions.clickSeeAllAttractions(driver)) throw `${city} failed`;
} }
if (seeAllUrl.length == 0) throw `${city} failed`;
await WebDriverUtils.wait(); await WebDriverUtils.wait();
console.log("Switch tab") console.log("Switch tab")
@ -69,7 +65,7 @@ async function visitCityPages(): Promise<void> {
} }
console.log("Click See all attractions") console.log("Click See all attractions")
if (!await UIActions.getSeeAllAttractionsUrl(driver)) throw `${city} failed`; if (!await UIActions.clickSeeAllAttractions(driver)) throw `${city} failed`;
await WebDriverUtils.wait(); await WebDriverUtils.wait();
console.log("Switch tab to Attraction") console.log("Switch tab to Attraction")

4
src/lib/UIActions.ts
View File

@ -80,7 +80,7 @@ export async function gotoHome(driver: WebDriver): Promise<boolean> {
} }
export async function clickSeeAllAttractions(driver: WebDriver): Promise<boolean> { export async function clickSeeAllAttractions(driver: WebDriver): Promise<boolean> {
const xpath = `//h2[starts-with(normalize-space(.), 'Top Attractions in')]/parent::*[1]//a[starts-with(@href, '/Attractions') and .//span[normalize-space(.)='See all']]`; const xpath = `//h3[starts-with(normalize-space(.), 'Things to do')]/parent::*[1]//a[starts-with(@href, '/Attractions') and .//span[normalize-space(.)='See all']]`;
try { try {
const anchorElement = await driver.wait(until.elementLocated(By.xpath(xpath)), 5000); const anchorElement = await driver.wait(until.elementLocated(By.xpath(xpath)), 5000);
@ -98,7 +98,7 @@ export async function clickSeeAllAttractions(driver: WebDriver): Promise<boolean
} }
export async function getSeeAllAttractionsUrl(driver: WebDriver): Promise<string | null> { export async function getSeeAllAttractionsUrl(driver: WebDriver): Promise<string | null> {
const xpath = `//h2[starts-with(normalize-space(.), 'Top Attractions in')]/parent::*[1]//a[starts-with(@href, '/Attractions') and .//span[normalize-space(.)='See all']]`; const xpath = `//h3[starts-with(normalize-space(.), 'Top Attractions in')]/parent::*[1]//a[starts-with(@href, '/Attractions') and .//span[normalize-space(.)='See all']]`;
try { try {
const anchorElement = await driver.wait(until.elementLocated(By.xpath(xpath)), 5000); const anchorElement = await driver.wait(until.elementLocated(By.xpath(xpath)), 5000);

20
src/lib/utils.ts
View File

@ -17,30 +17,24 @@ export class WebDriverUtils {
static async wait(driver?: WebDriver): Promise<void> { static async wait(driver?: WebDriver): Promise<void> {
const seconds = Math.floor(Math.random() * 1000) % 3 + 3; const seconds = Math.floor(Math.random() * 1000) % 3 + 3;
console.log(`Scrolling to bottom for ${seconds} seconds...`); console.log(`Scrolling to bottom for ${seconds} seconds...`);
const endTime = Date.now() + seconds * 1000; const endTime = Date.now() + seconds * 1000;
let scrollCounter = 0; let scrollCounter = 0;
while (Date.now() < endTime) { while (Date.now() < endTime) {
try { try {
if(driver){ if (driver) {
if(scrollCounter < 4){ await driver.executeScript(`
await driver.executeScript(` window.scrollBy(0, 10);
window.scrollBy(0, window.innerHeight); `);
`);
}else{
await driver.executeScript(`
window.scrollTo(0, 0);
`);
}
scrollCounter++; scrollCounter++;
} }
} catch (error) { } catch (error) {
console.warn('Scroll failed:', error); console.warn('Scroll failed:', error);
} }
// Wait a little between scrolls // Wait a little between scrolls
await new Promise(resolve => setTimeout(resolve, 500)); await new Promise(resolve => setTimeout(resolve, 500));
} }