ken/tripadviser_scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

save work

Browse Source
This commit is contained in:
Ken Yasue
2025-03-24 19:00:26 +01:00
parent fe177abd85
commit 47134525b8
3 changed files with 13 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/index.ts
View File

@ -23,7 +23,7 @@ async function visitCityPages(): Promise<void> {
console.log('Connecting to existing Chrome browser...');
const driver = await useChrome();
const driver = await useExistingChrome();
if (!driver) return;
// Visit each city's TripAdvisor page
@ -48,14 +48,10 @@ async function visitCityPages(): Promise<void> {
await WebDriverUtils.wait(driver);
console.log("Click See all")
let seeAllUrl = await UIActions.getSeeAllUrl(driver);
if (seeAllUrl.length == 0) {
if (!await UIActions.clickSeeAllAttractions(driver)) {
if (!await UIActions.clickTourismLink(driver)) throw `${city} failed`;
seeAllUrl = await UIActions.getSeeAllUrl(driver);
if (!await UIActions.clickSeeAllAttractions(driver)) throw `${city} failed`;
}
if (seeAllUrl.length == 0) throw `${city} failed`;
await WebDriverUtils.wait();
console.log("Switch tab")
@ -69,7 +65,7 @@ async function visitCityPages(): Promise<void> {
}
console.log("Click See all attractions")
if (!await UIActions.getSeeAllAttractionsUrl(driver)) throw `${city} failed`;
if (!await UIActions.clickSeeAllAttractions(driver)) throw `${city} failed`;
await WebDriverUtils.wait();
console.log("Switch tab to Attraction")

4
src/lib/UIActions.ts
View File

@ -80,7 +80,7 @@ export async function gotoHome(driver: WebDriver): Promise<boolean> {
}
export async function clickSeeAllAttractions(driver: WebDriver): Promise<boolean> {
const xpath = `//h2[starts-with(normalize-space(.), 'Top Attractions in')]/parent::*[1]//a[starts-with(@href, '/Attractions') and .//span[normalize-space(.)='See all']]`;
const xpath = `//h3[starts-with(normalize-space(.), 'Things to do')]/parent::*[1]//a[starts-with(@href, '/Attractions') and .//span[normalize-space(.)='See all']]`;
try {
const anchorElement = await driver.wait(until.elementLocated(By.xpath(xpath)), 5000);
@ -98,7 +98,7 @@ export async function clickSeeAllAttractions(driver: WebDriver): Promise<boolean
}
export async function getSeeAllAttractionsUrl(driver: WebDriver): Promise<string | null> {
const xpath = `//h2[starts-with(normalize-space(.), 'Top Attractions in')]/parent::*[1]//a[starts-with(@href, '/Attractions') and .//span[normalize-space(.)='See all']]`;
const xpath = `//h3[starts-with(normalize-space(.), 'Top Attractions in')]/parent::*[1]//a[starts-with(@href, '/Attractions') and .//span[normalize-space(.)='See all']]`;
try {
const anchorElement = await driver.wait(until.elementLocated(By.xpath(xpath)), 5000);

20
src/lib/utils.ts
View File

@ -17,30 +17,24 @@ export class WebDriverUtils {
static async wait(driver?: WebDriver): Promise<void> {
const seconds = Math.floor(Math.random() * 1000) % 3 + 3;
console.log(`Scrolling to bottom for ${seconds} seconds...`);
const endTime = Date.now() + seconds * 1000;
let scrollCounter = 0;
while (Date.now() < endTime) {
try {
if(driver){
if(scrollCounter < 4){
await driver.executeScript(`
window.scrollBy(0, window.innerHeight);
`);
}else{
await driver.executeScript(`
window.scrollTo(0, 0);
`);
}
if (driver) {
await driver.executeScript(`
window.scrollBy(0, 10);
`);
scrollCounter++;
}
} catch (error) {
console.warn('Scroll failed:', error);
}
// Wait a little between scrolls
await new Promise(resolve => setTimeout(resolve, 500));
}