wip

data/cities.csv

@@ -1,4 +1,20 @@
 rank,Latitude,Longitude,Name of City,Country,2021 Population,2020 Population,Growth,Population Difference,Population Change
+8,39.906217,116.3912757,Beijing,China,20896820,20462610,0.0212,434210,grew
+9,19.0759899,72.8773928,Mumbai,India,20667656,20411274,0.0126,256382,grew
+10,34.6198813,135.490357,Osaka,Japan,19110616,19165340,-0.0029,54724,declined
+11,24.8546842,67.0207055,Karachi,Pakistan,16459472,16093786,0.0227,365686,grew
+12,29.5647398,106.5478767,Chongqing,China,16382376,15872179,0.0321,510197,grew
+13,41.0096334,28.9651646,Istanbul,Turkey,15415197,15190336,0.0148,224861,grew
+14,-34.6075682,-58.4370894,Buenos Aires,Argentina,15257673,15153729,0.0069,103944,grew
+15,22.5414185,88.3576912,Kolkata,India,14974073,14850066,0.0084,124007,grew
+16,-4.3217055,15.3125974,Kinshasa,DR Congo,14970460,14342439,0.0438,628021,grew
+17,6.4550575,3.3941795,Lagos,Nigeria,14862111,14368332,0.0344,493779,grew
+18,14.5907332,120.9809674,Manila,Philippines,14158573,13923452,0.0169,235121,grew
+19,39.0856735,117.1951073,Tianjin,China,13794450,13589078,0.0151,205372,grew
+20,23.1301964,113.2592945,Guangzhou,China,13635397,13301532,0.0251,333865,grew
+21,-22.9110137,-43.2093727,Rio de Janeiro,Brazil,13544462,13458075,0.0064,86387,grew
+22,31.5656822,74.3141829,Lahore,Pakistan,13095166,12642423,0.0358,452743,grew
+23,12.9767936,77.590082,Bangalore,India,12764935,12326532,0.0356,438403,grew
 24,55.7504461,37.6174943,Moscow,Russia,12593252,12537954,0.0044,55298,grew
 25,22.555454,114.0543297,Shenzhen,China,12591696,12356820,0.019,234876,grew
 26,13.0836939,80.270186,Chennai,India,11235018,10971108,0.0241,263910,grew

src/index.ts

@@ -28,19 +28,22 @@ async function visitCityPages(): Promise<void> {
 
     // Visit each city's TripAdvisor page
     for (let i = 0; i < cities.length; i++) {
+
         const city = cities[i];
         console.log(`[${i + 1}/${cities.length}] Visiting TripAdvisor page for ${city}...`);
 
         try {
 
             const originalWindow = await driver.getWindowHandle();
+            await driver.get("https://www.tripadvisor.com/");
+            await WebDriverUtils.wait(driver);
 
             console.log("Logo click")
             if (!await UIActions.gotoHome(driver)) throw `${city} failed`;
             await WebDriverUtils.wait(driver);
 
             console.log("Exec Search")
-            if (!await UIActions.execSearch(driver, `${city} museum`)) throw `${city} failed`;
+            if (!await UIActions.execSearch(driver, `"${city}" museums`)) throw `${city} failed`;
             await WebDriverUtils.wait(driver);
 
             let page = 1;
@@ -63,29 +66,35 @@ async function visitCityPages(): Promise<void> {
 
             }
 
-            for (const museumName of museumNames) {
+            for (const museumIndex in museumNames) {
 
-                console.log("back to home");
-                if (!await UIActions.gotoHome(driver)) throw `${city} failed`;
-                await WebDriverUtils.wait(driver);
+                const museumName = museumNames[museumIndex];
 
-                if (!await UIActions.typeSearch(driver, `${city} ${museumName}`)) throw `${city} failed`;
-                await WebDriverUtils.wait(driver);
+                try{
+                    console.log(`reading museum ${museumIndex}/${museumNames.length}`);
     
-                if (!await UIActions.clickFirstAttractionLinkInForm(driver)) throw `${city} failed`;
-                await WebDriverUtils.wait(driver);
+                    if (!await UIActions.gotoHome(driver)) throw `${city} failed`;
+                    await WebDriverUtils.wait(driver);
     
-                const { websiteUrl, email } = await UIActions.getWebsiteAndEmail(driver);
+                    if (!await UIActions.typeSearch(driver, `${city} ${museumName}`)) throw `${city} failed`;
+                    await WebDriverUtils.wait(driver);
     
-                console.log(`${websiteUrl} / ${email}`);
-                saveContactInfoToCSV(city, { websiteUrl: websiteUrl, email: email }, path.join(__dirname, '../data/contact_info.csv'));
+                    await UIActions.clickFirstAttractionLinkInForm(driver)
+                    await WebDriverUtils.wait(driver);
+    
+                    const { websiteUrl, email } = await UIActions.getWebsiteAndEmail(driver);
+    
+                    console.log(`${websiteUrl} / ${email}`);
+                    saveContactInfoToCSV(city, { name: museumName, websiteUrl: websiteUrl, email: email }, path.join(__dirname, '../data/contact_info.csv'));
+    
+                    await UIActions.simulateClickAt(driver,100,100);
+                    await WebDriverUtils.wait(driver);
+                }catch(e){
+                    console.error(`failed ${museumName}`)
+                }
                
             }
 
-            console.log(museumNames);
-            await UIActions.closeAllTabsExceptFirst(driver);
-
-
             if (i < cities.length - 1) {
                 console.log(`Waiting for 5000 seconds before next city...`);
                 await WebDriverUtils.wait(); // Wait 5000 seconds before next city

src/lib/UIActions.ts

@@ -1,4 +1,4 @@
-import { Builder, By, until, WebDriver, WebElement } from 'selenium-webdriver';
+import { Builder, By, until, WebDriver, WebElement, Actions } from 'selenium-webdriver';
 import * as chromedriver from 'chromedriver';
 import chrome, { ServiceBuilder } from 'selenium-webdriver/chrome';
 import * as fs from 'fs';
@@ -224,7 +224,7 @@ export async function getAttractionNames(driver: WebDriver): Promise<string[]> {
 }
 
 export async function getWebsiteAndEmail(driver: WebDriver): Promise<ContactInfo> {
-    const result: ContactInfo = { websiteUrl: null, email: null };
+    const result: ContactInfo = { name: null, websiteUrl: null, email: null };
 
     // XPath to find URL (starting with 'http' but not containing 'tripadvisor')
     const urlXPath = `//a[starts-with(@href, 'http') and not(contains(@href, 'tripadvisor'))]`;
@@ -307,3 +307,21 @@ export async function clickFirstAttractionLinkInForm(driver: WebDriver): Promise
         return false;
     }
 }
+
+export async function simulateClickAt(driver: WebDriver, x: number, y: number): Promise<void> {
+    try {
+
+        const bodyElement = await driver.findElement(By.css('body'));
+
+        await driver
+            .actions({ bridge: true }) // optional; required in some environments
+            .move({ x, y, origin: bodyElement }) // move relative to the viewport
+            .click()
+            .perform();
+
+        console.log(`Clicked at (${x}, ${y})`);
+    } catch (error) {
+        console.error('An error occurred:', error);
+    }
+}
+

src/lib/types.ts

@@ -1,4 +1,5 @@
 export interface ContactInfo {
+    name: string | null,
     websiteUrl: string | null;
     email: string | null;
 }

src/lib/utils.ts

@@ -15,7 +15,7 @@ export class WebDriverUtils {
      * @returns Promise that resolves after the specified time
      */
     static async wait(driver?: WebDriver): Promise<void> {
-        const seconds = Math.floor(Math.random() * 1000) % 3 + 3;
+        const seconds = Math.floor(Math.random() * 1000) % 3 + 2;
         console.log(`Scrolling to bottom for ${seconds} seconds...`);
 
         const endTime = Date.now() + seconds * 1000;
@@ -58,7 +58,7 @@ export class WebDriverUtils {
 
 export function saveContactInfoToCSV(city: string, contactInfo: ContactInfo, filePath: string): void {
     const headers = 'City,Website URL,Email\n';
-    const line = `"${city},"${contactInfo.websiteUrl}","${contactInfo.email}"\n`;
+    const line = `"${city}","${contactInfo.name}","${contactInfo.websiteUrl}","${contactInfo.email}"\n`;
 
     if (!existsSync(filePath)) {
         writeFileSync(filePath, headers + line);