156 lines
5.6 KiB
TypeScript
156 lines
5.6 KiB
TypeScript
/**
|
|
* Selenium WebDriver script to visit TripAdvisor pages for random cities
|
|
* & 'C:\Program Files\Google\Chrome\Application\chrome.exe' --remote-debugging-port=9222
|
|
*/
|
|
|
|
import { Builder, By, until, WebDriver } from 'selenium-webdriver';
|
|
import * as chromedriver from 'chromedriver';
|
|
import chrome, { ServiceBuilder } from 'selenium-webdriver/chrome';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import { getCities } from './lib/cities';
|
|
import { WebDriverUtils, saveContactInfoToCSV, useExistingChrome, disableCookiesInChrome } from './lib/utils';
|
|
import * as UIActions from './lib/UIActions';
|
|
import { randomUUID } from 'crypto';
|
|
|
|
|
|
/**
|
|
* Function to visit TripAdvisor pages for each city
|
|
*/
|
|
async function visitCityPages(): Promise<void> {
|
|
|
|
const cities = getCities(path.join(__dirname, '../data/cities.csv'));
|
|
|
|
console.log('Connecting to existing Chrome browser...');
|
|
|
|
const driver = await useExistingChrome();
|
|
if (!driver) return;
|
|
|
|
// Visit each city's TripAdvisor page
|
|
for (let i = 0; i < cities.length; i++) {
|
|
const city = cities[i];
|
|
console.log(`[${i + 1}/${cities.length}] Visiting TripAdvisor page for ${city}...`);
|
|
let originalWindow;
|
|
let cityTopWindow;
|
|
let attactionsWindow;
|
|
let museumWindow;
|
|
|
|
try {
|
|
|
|
const originalWindow = await driver.getWindowHandle();
|
|
|
|
console.log("Logo click")
|
|
if (!await UIActions.gotoHome(driver)) throw `${city} failed`;
|
|
await WebDriverUtils.wait(driver);
|
|
|
|
console.log("Exec Search")
|
|
if (!await UIActions.execSearch(driver, city)) throw `${city} failed`;
|
|
await WebDriverUtils.wait(driver);
|
|
|
|
console.log("Click See all")
|
|
if (!await UIActions.clickSeeAll(driver)) {
|
|
if (!await UIActions.clickTourismLink(driver)) throw `${city} failed`;
|
|
if (!await UIActions.clickSeeAll(driver)) throw `${city} failed`;
|
|
}
|
|
await WebDriverUtils.wait(driver);
|
|
|
|
console.log("Switch tab")
|
|
let windows = await driver.getAllWindowHandles();
|
|
// Switch to the newly opened window/tab
|
|
for (const handle of windows) {
|
|
if (handle !== originalWindow) {
|
|
cityTopWindow = handle;
|
|
await driver.switchTo().window(handle);
|
|
}
|
|
}
|
|
|
|
console.log("Click See all attractions")
|
|
if (!await UIActions.clickSeeAllAttractions(driver)) throw `${city} failed`;
|
|
await WebDriverUtils.wait(driver);
|
|
|
|
console.log("Switch tab to Attraction")
|
|
windows = await driver.getAllWindowHandles();
|
|
// Switch to the newly opened window/tab
|
|
for (const handle of windows) {
|
|
if (handle !== originalWindow && handle !== cityTopWindow) {
|
|
attactionsWindow = handle;
|
|
await driver.switchTo().window(attactionsWindow);
|
|
}
|
|
}
|
|
|
|
// click museum
|
|
console.log("Click Museum link");
|
|
if (!await UIActions.clickMuseumsLink(driver)) throw `${city} failed`;
|
|
await WebDriverUtils.wait(driver);
|
|
|
|
let page = 1;
|
|
while (1) {
|
|
|
|
// get list of museums
|
|
console.log("Get list of museums");
|
|
const museumElms = await UIActions.getMusiums(driver);
|
|
await WebDriverUtils.wait(driver);
|
|
|
|
for (const listItem of museumElms) {
|
|
|
|
await listItem.click();
|
|
await WebDriverUtils.wait(driver);
|
|
|
|
windows = await driver.getAllWindowHandles();
|
|
for (const handle of windows) {
|
|
if (handle !== originalWindow && handle !== cityTopWindow && handle !== attactionsWindow) {
|
|
museumWindow = handle;
|
|
await driver.switchTo().window(museumWindow);
|
|
}
|
|
}
|
|
|
|
const { websiteUrl, email } = await UIActions.getWebsiteAndEmail(driver);
|
|
|
|
console.log(`${websiteUrl} / ${email}`);
|
|
saveContactInfoToCSV(city, { websiteUrl: websiteUrl, email: email }, path.join(__dirname, '../data/contact_info.csv'));
|
|
|
|
museumWindow && await driver.switchTo().window(museumWindow);
|
|
await driver.close();
|
|
await WebDriverUtils.wait(driver);
|
|
|
|
attactionsWindow && await driver.switchTo().window(attactionsWindow);
|
|
await WebDriverUtils.wait(driver);
|
|
|
|
}
|
|
|
|
page++;
|
|
|
|
if (page > 10) break;
|
|
|
|
UIActions.clickPagination(driver, page);
|
|
await WebDriverUtils.wait(driver);
|
|
|
|
}
|
|
|
|
|
|
await UIActions.closeAllTabsExceptFirst(driver);
|
|
|
|
|
|
if (i < cities.length - 1) {
|
|
console.log(`Waiting for 5000 seconds before next city...`);
|
|
await WebDriverUtils.wait(); // Wait 5000 seconds before next city
|
|
}
|
|
} catch (error) {
|
|
|
|
await UIActions.closeAllTabsExceptFirst(driver);
|
|
|
|
// If the button is not found within the timeout, log and continue to the next city
|
|
console.log(`No Museums button found for ${city}. Moving to next city after 5 seconds...`);
|
|
await WebDriverUtils.wait(); // Wait 5 seconds before next city
|
|
}
|
|
}
|
|
|
|
console.log('Finished visiting all cities!');
|
|
|
|
}
|
|
|
|
// Run the function
|
|
visitCityPages().catch(error => {
|
|
console.error('Error in main function:', error);
|
|
});
|