Merge branch 'main' of https://git.yasue.org/ken/tripadviser_scraper

src/index.ts

@@ -9,7 +9,7 @@ import chrome, { ServiceBuilder } from 'selenium-webdriver/chrome';
 import * as fs from 'fs';
 import * as path from 'path';
 import { getCities } from './lib/cities';
-import { WebDriverUtils, saveContactInfoToCSV, useExistingChrome, disableCookiesInChrome, useChrome } from './lib/utils';
+import { WebDriverUtils, saveContactInfoToCSV, useExistingChrome, disableCookiesInChrome, useChrome, saveMuseumnameToCSV } from './lib/utils';
 import * as UIActions from './lib/UIActions';
 import { randomUUID } from 'crypto';
 
@@ -47,15 +47,15 @@ async function visitCityPages(): Promise<void> {
             await WebDriverUtils.wait(driver);
 
             let page = 1;
-            let museumNames: string[] = [];
 
             while (1) {
 
                 // get list of museums
                 console.log("Get list of museums");
                 const tmpMuseumNames = await UIActions.getAttractionNames(driver);
-                await WebDriverUtils.wait(driver);
+
-                museumNames = [...museumNames, ...tmpMuseumNames]
+                if (tmpMuseumNames.length == 0)
+                    break;
 
                 page++;
 
@@ -64,8 +64,13 @@ async function visitCityPages(): Promise<void> {
                 UIActions.clickPagination(driver, page);
                 await WebDriverUtils.wait(driver);
 
+                tmpMuseumNames.forEach((museumName) => {
+                    saveMuseumnameToCSV(city, museumName, path.join(__dirname, '../data/museums.csv'));
+                })
+
             }
 
+            /*
             for (const museumIndex in museumNames) {
 
                 const museumName = museumNames[museumIndex];
@@ -94,6 +99,7 @@ async function visitCityPages(): Promise<void> {
                 }
                
             }
+            */
 
             if (i < cities.length - 1) {
                 console.log(`Waiting for 5000 seconds before next city...`);

src/lib/utils.ts

@@ -69,6 +69,20 @@ export function saveContactInfoToCSV(city: string, contactInfo: ContactInfo, fil
     console.log(`Contact info saved to ${filePath}`);
 }
 
+export function saveMuseumnameToCSV(city: string, museumName: string, filePath: string): void {
+    const headers = 'City,Website URL,Email\n';
+    const line = `"${city},"${museumName}"\n`;
+
+    if (!existsSync(filePath)) {
+        writeFileSync(filePath, headers + line);
+    } else {
+        appendFileSync(filePath, line);
+    }
+
+    console.log(`Museum name saved to ${filePath}`);
+}
+
+
 export async function disableCookiesInChrome(): Promise<WebDriver | null> {
     // Set Chrome options
     const options = new chrome.Options();