ken/tripadviser_scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new way of scraping

Browse Source
This commit is contained in:
Ken Yasue
2025-03-24 20:24:30 +01:00
parent 47134525b8
commit 6153b70c1e
4 changed files with 127 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/lib/utils.ts
View File

@ -23,6 +23,8 @@ export class WebDriverUtils {
let scrollCounter = 0;
while (Date.now() < endTime) {
/*
try {
if (driver) {
await driver.executeScript(`
@ -34,6 +36,7 @@ export class WebDriverUtils {
} catch (error) {
console.warn('Scroll failed:', error);
}
*/
// Wait a little between scrolls
await new Promise(resolve => setTimeout(resolve, 500));
@ -105,7 +108,7 @@ export async function disableCookiesInChrome(): Promise<WebDriver | null> {
}
export async function useChrome(): Promise<WebDriver | null> {
export async function useChrome(initialUrl: string = "https://www.tripadvisor.com/"): Promise<WebDriver | null> {
// Set Chrome options
const options = new chrome.Options();
@ -127,7 +130,7 @@ export async function useChrome(): Promise<WebDriver | null> {
.setChromeOptions(options)
.build();
await driver.get('https://www.tripadvisor.com/');
await driver.get(initialUrl);
console.log('Chrome launched with cookies disabled.');