ken/tripadviser_scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

save work

Browse Source
This commit is contained in:
Ken Yasue
2025-03-24 19:00:26 +01:00
parent fe177abd85
commit 47134525b8
3 changed files with 13 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/index.ts
View File

@ -23,7 +23,7 @@ async function visitCityPages(): Promise<void> {
console.log('Connecting to existing Chrome browser...'); console.log('Connecting to existing Chrome browser...');
const driver = await useChrome(); const driver = await useExistingChrome();
if (!driver) return; if (!driver) return;
// Visit each city's TripAdvisor page // Visit each city's TripAdvisor page
@ -48,14 +48,10 @@ async function visitCityPages(): Promise<void> {
await WebDriverUtils.wait(driver); await WebDriverUtils.wait(driver);
console.log("Click See all") console.log("Click See all")
let seeAllUrl = await UIActions.getSeeAllUrl(driver); if (!await UIActions.clickSeeAllAttractions(driver)) {
if (seeAllUrl.length == 0) {
if (!await UIActions.clickTourismLink(driver)) throw `${city} failed`; if (!await UIActions.clickTourismLink(driver)) throw `${city} failed`;
seeAllUrl = await UIActions.getSeeAllUrl(driver); if (!await UIActions.clickSeeAllAttractions(driver)) throw `${city} failed`;
} }
if (seeAllUrl.length == 0) throw `${city} failed`;
await WebDriverUtils.wait(); await WebDriverUtils.wait();
console.log("Switch tab") console.log("Switch tab")
@ -69,7 +65,7 @@ async function visitCityPages(): Promise<void> {
} }
console.log("Click See all attractions") console.log("Click See all attractions")
if (!await UIActions.getSeeAllAttractionsUrl(driver)) throw `${city} failed`; if (!await UIActions.clickSeeAllAttractions(driver)) throw `${city} failed`;
await WebDriverUtils.wait(); await WebDriverUtils.wait();
console.log("Switch tab to Attraction") console.log("Switch tab to Attraction")

4
src/lib/UIActions.ts
View File

@ -80,7 +80,7 @@ export async function gotoHome(driver: WebDriver): Promise<boolean> {
} }
export async function clickSeeAllAttractions(driver: WebDriver): Promise<boolean> { export async function clickSeeAllAttractions(driver: WebDriver): Promise<boolean> {
const xpath = `//h2[starts-with(normalize-space(.), 'Top Attractions in')]/parent::*[1]//a[starts-with(@href, '/Attractions') and .//span[normalize-space(.)='See all']]`; const xpath = `//h3[starts-with(normalize-space(.), 'Things to do')]/parent::*[1]//a[starts-with(@href, '/Attractions') and .//span[normalize-space(.)='See all']]`;
try { try {
const anchorElement = await driver.wait(until.elementLocated(By.xpath(xpath)), 5000); const anchorElement = await driver.wait(until.elementLocated(By.xpath(xpath)), 5000);
@ -98,7 +98,7 @@ export async function clickSeeAllAttractions(driver: WebDriver): Promise<boolean
} }
export async function getSeeAllAttractionsUrl(driver: WebDriver): Promise<string | null> { export async function getSeeAllAttractionsUrl(driver: WebDriver): Promise<string | null> {
const xpath = `//h2[starts-with(normalize-space(.), 'Top Attractions in')]/parent::*[1]//a[starts-with(@href, '/Attractions') and .//span[normalize-space(.)='See all']]`; const xpath = `//h3[starts-with(normalize-space(.), 'Top Attractions in')]/parent::*[1]//a[starts-with(@href, '/Attractions') and .//span[normalize-space(.)='See all']]`;
try { try {
const anchorElement = await driver.wait(until.elementLocated(By.xpath(xpath)), 5000); const anchorElement = await driver.wait(until.elementLocated(By.xpath(xpath)), 5000);

10
src/lib/utils.ts
View File

@ -24,16 +24,10 @@ export class WebDriverUtils {
while (Date.now() < endTime) { while (Date.now() < endTime) {
try { try {
if(driver){ if (driver) {
if(scrollCounter < 4){
await driver.executeScript(` await driver.executeScript(`
window.scrollBy(0, window.innerHeight); window.scrollBy(0, 10);
`); `);
}else{
await driver.executeScript(`
window.scrollTo(0, 0);
`);
}
scrollCounter++; scrollCounter++;
} }