Merge branch 'master' of https://git.yasue.org/ken/RandomVideoMaker

.gitignore

@@ -22,3 +22,4 @@ yarn-error.log*
 /download/
 /generated/
 .env
+input/

src/lib/lmstudio.ts

@@ -4,25 +4,27 @@ import { logger } from './logger';
 
 dotenv.config();
 
-const LLM_BASE_URL = process.env.LLM_BASE_URL;
+const LMSTUDIO_BASE_URL = process.env.LMSTUDIO_BASE_URL;
+const LMSTUDIO_API_KEY = process.env.LMSTUDIO_API_KEY;
+const LMSTUDIO_MODEL = process.env.LMSTUDIO_MODEL;
 
-async function callLMStudio(prompt: string): Promise<any> {
+async function callLmstudio(prompt: string): Promise<any> {
-    if (!LLM_BASE_URL) {
+    if (!LMSTUDIO_BASE_URL) {
-        throw new Error('LLM_BASE_URL is not defined in the .env file');
+        throw new Error('LMSTUDIO_BASE_URL is not defined in the .env file');
     }
 
     for (let i = 0; i < 10; i++) {
         let llmResponse = "";
 
         try {
-            const requestUrl = new URL('v1/chat/completions', LLM_BASE_URL);
+            const response = await fetch(`${LMSTUDIO_BASE_URL}/chat/completions`, {
-            const response = await fetch(requestUrl, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${LMSTUDIO_API_KEY}`,
                 },
                 body: JSON.stringify({
-                    model: 'local-model',
+                    model: LMSTUDIO_MODEL,
                     messages: [
                         {
                             role: 'user',
@@ -40,15 +42,19 @@ async function callLMStudio(prompt: string): Promise<any> {
                 const jsonMatch = content.match(/\{[\s\S]*\}/);
                 if (jsonMatch) {
                     return JSON.parse(jsonMatch[0]);
+                } else {
+                    const arrayMatch = content.match(/\[[\s\S]*\]/);
+                    if (arrayMatch) {
+                        return JSON.parse(arrayMatch[0]);
+                    }
                 }
+                // If no JSON/array found, return the raw content
+                return content;
             } else {
                 logger.error('Unexpected API response:', data);
             }
         } catch (error) {
             logger.error(`Attempt ${i + 1} failed:`, error);
-            if (error instanceof TypeError && error.message.includes('fetch failed')) {
-                logger.error('Could not connect to the LM Studio server. Please ensure the server is running and accessible at the specified LLM_BASE_URL.');
-            }
             logger.debug(`LLM response: ${llmResponse}`)
         }
     }
@@ -56,9 +62,9 @@ async function callLMStudio(prompt: string): Promise<any> {
     throw new Error('Failed to get response from LLM after 10 attempts');
 }
 
-async function callLMStudioWithFile(imagePath: string, prompt: string): Promise<any> {
+async function callLMStudioAPIWithFile(imagePath: string, prompt: string): Promise<any> {
-    if (!LLM_BASE_URL) {
+    if (!LMSTUDIO_BASE_URL) {
-        throw new Error('LLM_BASE_URL is not defined in the .env file');
+        throw new Error('LMSTUDIO_BASE_URL is not defined in the .env file');
     }
 
     const imageBuffer = fs.readFileSync(imagePath);
@@ -68,14 +74,14 @@ async function callLMStudioWithFile(imagePath: string, prompt: string): Promise<
         let llmResponse = "";
 
         try {
-            const requestUrl = new URL('v1/chat/completions', LLM_BASE_URL);
+            const response = await fetch(`${LMSTUDIO_BASE_URL}/chat/completions`, {
-            const response = await fetch(requestUrl, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
+                    'Authorization': `Bearer ${LMSTUDIO_API_KEY}`,
                 },
                 body: JSON.stringify({
-                    model: 'local-model',
+                    model: LMSTUDIO_MODEL,
                     messages: [
                         {
                             role: 'user',
@@ -96,15 +102,17 @@ async function callLMStudioWithFile(imagePath: string, prompt: string): Promise<
                 const jsonMatch = content.match(/\{[\s\S]*\}/);
                 if (jsonMatch) {
                     return JSON.parse(jsonMatch[0]);
+                } else {
+                    const arrayMatch = content.match(/\[[\s\S]*\]/);
+                    if (arrayMatch) {
+                        return JSON.parse(arrayMatch[0]);
+                    }
                 }
             } else {
                 logger.error('Unexpected API response:', data);
             }
         } catch (error) {
             logger.error(`Attempt ${i + 1} failed:`, error);
-            if (error instanceof TypeError && error.message.includes('fetch failed')) {
-                logger.error('Could not connect to the LM Studio server. Please ensure the server is running and accessible at the specified LLM_BASE_URL.');
-            }
             logger.debug(`LLM response: ${llmResponse}`)
         }
     }
@@ -112,4 +120,4 @@ async function callLMStudioWithFile(imagePath: string, prompt: string): Promise<
     throw new Error('Failed to describe image after 10 attempts');
 }
 
-export { callLMStudio, callLMStudioWithFile };
+export { callLmstudio, callLMStudioAPIWithFile };

src/musicspot_generator/v1/images.ts

@@ -2,11 +2,11 @@ import dotenv from 'dotenv';
 import path from 'path';
 import fs from 'fs/promises';
 
-import { logger } from '../lib/logger';
+import { logger } from '../../lib/logger';
-import { callOpenAI } from '../lib/openai';
+import { callOpenAI } from '../../lib/openai';
-import { callLMStudio } from '../lib/lmstudio';
+import { callLMStudio } from '../../lib/lmstudio';
 // import { generateImage as generateFaceImage } from '../lib/image-generator-face'; // Removed
-import { generateImage } from '../lib/image-generator'; // Added
+import { generateImage } from '../../lib/image-generator'; // Added
 
 dotenv.config();
 

src/musicspot_generator/v1/index.ts

@@ -2,10 +2,10 @@ import dotenv from 'dotenv';
 import path from 'path';
 import fs from 'fs/promises';
 
-import { logger } from '../lib/logger';
+import { logger } from '../../lib/logger';
-import { callOpenAI } from '../lib/openai';
+import { callOpenAI } from '../../lib/openai';
-import { generateImage as generateFaceImage } from '../lib/image-generator-face';
+import { generateImage as generateFaceImage } from '../../lib/image-generator-face';
-import { generateVideo } from '../lib/video-generator';
+import { generateVideo } from '../../lib/video-generator';
 
 dotenv.config();
 

src/musicspot_generator/v1/videos.ts

@@ -2,9 +2,9 @@ import dotenv from 'dotenv';
 import path from 'path';
 import fs from 'fs/promises';
 
-import { logger } from '../lib/logger';
+import { logger } from '../../lib/logger';
-import { callOpenAI } from '../lib/openai';
+import { callOpenAI } from '../../lib/openai';
-import { generateVideo } from '../lib/video-generator';
+import { generateVideo } from '../../lib/video-generator';
 
 dotenv.config();
 
@@ -234,7 +234,7 @@ async function main() {
                             serverForVideo.baseUrl!,
                             serverForVideo.outputDir!,
                             DEFAULT_SIZE,
-                            true,
+                            false,
                             false
                         );
 

src/musicspot_generator/v2/generate_photo.ts

@@ -0,0 +1,62 @@
+import fs from 'fs';
+import path from 'path';
+import crypto from 'crypto';
+import { generateImage } from '../../lib/image-generator';
+import { logger } from '../../lib/logger';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+const scenesFilePath = path.resolve(process.cwd(), 'src/musicspot_generator/v2/scenes.json');
+const GENERATED_DIR = path.resolve('generated');
+const DEFAULT_SIZE = { width: 1280, height: 720 };
+
+interface Scene {
+    scene: string;
+    imagePrompt: string;
+    videoPromp: string;
+    baseImagePath: string;
+}
+
+const COMFY_BASE_URL = process.env.COMFY_BASE_URL;
+const COMFY_OUTPUT_DIR = process.env.COMFY_OUTPUT_DIR;
+
+async function generatePhotos() {
+    if (!COMFY_BASE_URL || !COMFY_OUTPUT_DIR) {
+        throw new Error('COMFY_BASE_URL or COMFY_OUTPUT_DIR is not defined in the .env file');
+    }
+
+    const scenesFileContent = fs.readFileSync(scenesFilePath, 'utf-8');
+    const scenesData: { scenes: Scene[] } = JSON.parse(scenesFileContent);
+
+    for (const scene of scenesData.scenes) {
+        const hash = crypto.createHash('sha256').update(scene.baseImagePath).digest('hex');
+        const imgFileName = `${hash}.png`;
+        const outputFilePath = path.join(GENERATED_DIR, imgFileName);
+
+        if (fs.existsSync(outputFilePath)) {
+            logger.info(`Skipping already generated photo for: ${scene.baseImagePath}`);
+            continue;
+        }
+
+        logger.info(`Generating photo for: ${scene.baseImagePath}`);
+
+        try {
+            await generateImage(
+                scene.imagePrompt,
+                imgFileName,
+                COMFY_BASE_URL,
+                COMFY_OUTPUT_DIR,
+                'flux',
+                DEFAULT_SIZE
+            );
+            logger.info(`Successfully generated photo: ${imgFileName}`);
+        } catch (error) {
+            logger.error(`Error generating photo for scene ${scene.scene}:`, error);
+        }
+    }
+}
+
+generatePhotos().catch(error => {
+    logger.error('An unexpected error occurred:', error);
+});

src/musicspot_generator/v2/generate_scenes.ts

@@ -0,0 +1,87 @@
+import fs from 'fs';
+import path from 'path';
+import { callLMStudioAPIWithFile } from '../../lib/lmstudio';
+import { logger } from '../../lib/logger';
+
+const promptInstructions = `
+    Video prompt: No slowmotion, Be creative and generate dynamic dance scene.
+    `;
+
+const inputDir = path.resolve(process.cwd(), 'input');
+const outputFilePath = path.resolve(process.cwd(), 'src/musicspot_generator/v2/scenes.json');
+
+interface Scene {
+    scene: string;
+    imagePrompt: string;
+    videoPromp: string;
+    baseImagePath: string;
+}
+
+async function processImages() {
+    const imageFiles = fs.readdirSync(inputDir).filter(file => /\.(png|jpg|jpeg)$/i.test(file));
+    let scenes: { scenes: Scene[] } = { scenes: [] };
+
+    if (fs.existsSync(outputFilePath)) {
+        const fileContent = fs.readFileSync(outputFilePath, 'utf-8');
+        if (fileContent) {
+            scenes = JSON.parse(fileContent);
+        }
+    }
+
+    for (const imageFile of imageFiles) {
+        const imagePath = path.resolve(inputDir, imageFile);
+        const absoluteImagePath = path.resolve(imagePath);
+
+        const existingScene = scenes.scenes.find(s => s.baseImagePath === absoluteImagePath);
+        if (existingScene) {
+            logger.info(`Skipping already processed image: ${imageFile}`);
+            continue;
+        }
+
+        logger.info(`Processing image: ${imageFile} `);
+
+        const prompt = `
+            Analyze the provided image and generate a JSON object with the following structure:
+{
+    "scenes": [
+        {
+            "scene": "A descriptive title for the scene in the image.",
+            "imagePrompt": {
+                "description": "A detailed description of the image content.",
+                "style": "Art style or photography style of the image.",
+                "lighting": "Description of the lighting in the image.",
+                "outfit": "Description of the outfit or clothing style in the image.",
+                "location": "Description of the location or setting of the image.",
+                "poses": "Description of the poses or actions of any subjects in the image.",
+                "angle": "Description of the camera angle or perspective of the image.",
+            }
+            "videoPromp": "Based on the image, create a prompt for a video that shows what might happen next or brings the scene to life.",
+            "baseImagePath": "The absolute path of the base image."
+        }
+    ]
+}
+
+Instructions: ${promptInstructions}
+`;
+
+        try {
+            const result = await callLMStudioAPIWithFile(imagePath, prompt);
+            if (result && result.scenes) {
+                const newScene = result.scenes[0];
+                newScene.baseImagePath = absoluteImagePath; // Ensure the path is correct
+                scenes.scenes.push(newScene);
+
+                fs.writeFileSync(outputFilePath, JSON.stringify(scenes, null, 2));
+                logger.info(`Successfully processed and saved scene for: ${imageFile} `);
+            } else {
+                logger.error('Failed to get valid scene data from API for image:', imageFile);
+            }
+        } catch (error) {
+            logger.error(`Error processing image ${imageFile}: `, error);
+        }
+    }
+}
+
+processImages().catch(error => {
+    logger.error('An unexpected error occurred:', error);
+});

src/musicspot_generator/v2/generate_video.ts

@@ -0,0 +1,57 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+import { generateVideo } from '../../lib/video-generator';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+interface Scene {
+    scene: string;
+    videoPrompt: string;
+    baseImagePath: string;
+}
+
+const scenesFilePath = path.join(__dirname, 'scenes.json');
+const generatedFolderPath = path.join(__dirname, '..', '..', '..', 'generated');
+
+async function processScenes() {
+    try {
+        const scenesData = fs.readFileSync(scenesFilePath, 'utf-8');
+        const scenes: Scene[] = JSON.parse(scenesData).scenes;
+
+        for (const scene of scenes) {
+            const hash = crypto.createHash('sha256').update(scene.baseImagePath).digest('hex');
+            const imageFileName = `${hash}.png`;
+            const imagePath = path.join(generatedFolderPath, imageFileName);
+
+            if (fs.existsSync(imagePath)) {
+                const outputVideoFileName = `${hash}.mp4`;
+                const outputVideoPath = path.join(generatedFolderPath, outputVideoFileName);
+
+                if (fs.existsSync(outputVideoPath)) {
+                    console.log(`Video already exists for scene ${scene.scene}, skipping.`);
+                    continue;
+                }
+
+                console.log(`Generating video for scene ${scene.scene}...`);
+
+                await generateVideo(
+                    scene.videoPrompt,
+                    imagePath,
+                    outputVideoPath,
+                    process.env.COMFY_BASE_URL!,
+                    process.env.COMFY_OUTPUT_DIR!,
+                    { width: 1280, height: 720 }
+                );
+                console.log(`Video for scene ${scene.scene} saved to ${outputVideoPath}`);
+            } else {
+                console.warn(`Image not found for scene ${scene.scene}: ${imagePath}`);
+            }
+        }
+    } catch (error) {
+        console.error('Error processing scenes:', error);
+    }
+}
+
+processScenes();

src/musicspot_generator/v2/photo_download.ts

@@ -0,0 +1,227 @@
+import { callLmstudio } from '../../lib/lmstudio';
+import { logger } from '../../lib/logger';
+import * as fs from 'fs/promises';
+import dotenv from 'dotenv';
+import path from 'path';
+import puppeteer from 'puppeteer';
+
+dotenv.config();
+
+const SCROLL_SEARCH = 3; // scroll times on search results
+const SCROLL_PIN = 3; // scroll times on pin page
+const PINS_TO_COLLECT = 5;
+
+// Hard-coded user prompt
+const HARDCODED_USER_PROMPT = process.env.HARDCODED_USER_PROMPT || `
+    Generate 20 keywords for photos of group of people dancing together focus on street and urban style. All keywords shoudld contain \"group horizontal\" and what you create.
+    Example output : ["group horizontal hiphop dance","group horizontal modern dance","",... and 20 items in array]
+    `;
+
+async function getPinUrlsFromPinterest(keyword: string, scrollCount = SCROLL_SEARCH, limit = PINS_TO_COLLECT): Promise<string[]> {
+    const browser = await puppeteer.launch({ headless: true });
+    const page = await browser.newPage();
+    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36');
+    await page.setViewport({ width: 1920, height: 1080 });
+    try {
+        const searchUrl = `https://www.pinterest.com/search/pins/?q=${encodeURIComponent(keyword)}`;
+        await page.goto(searchUrl, { waitUntil: 'networkidle2' });
+
+        let pinLinks = new Set<string>();
+        for (let i = 0; i < scrollCount; i++) {
+            const linksBefore = pinLinks.size;
+            const newLinks = await page.$$eval('a', (anchors) =>
+                anchors.map((a) => a.href).filter((href) => href.includes('/pin/'))
+            );
+            newLinks.forEach(link => pinLinks.add(link));
+
+            if (pinLinks.size >= limit) {
+                break;
+            }
+
+            await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
+            await new Promise(r => setTimeout(r, 500 + Math.random() * 1000));
+
+            if (pinLinks.size === linksBefore) {
+                // If no new pins are loaded, stop scrolling
+                logger.info(`No new pins loaded for "${keyword}", stopping scroll.`);
+                break;
+            }
+        }
+        return Array.from(pinLinks).slice(0, limit);
+    } catch (error) {
+        logger.error(`Error while getting pin URLs from Pinterest for keyword "${keyword}":`, error);
+        return [];
+    } finally {
+        await browser.close();
+    }
+}
+
+async function downloadImagesFromPin(pinUrl: string, scrollTimes = SCROLL_PIN): Promise<string[]> {
+    const browser = await puppeteer.launch({ headless: true });
+    const page = await browser.newPage();
+    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36');
+    await page.setViewport({ width: 1920, height: 1080 });
+    try {
+        await page.goto(pinUrl, { waitUntil: 'networkidle2', timeout: 30000 });
+        for (let i = 0; i < scrollTimes; i++) {
+            await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
+            await new Promise((r) => setTimeout(r, 700 + Math.random() * 800));
+        }
+        const imgs: string[] = await page.$$eval('img', imgs => {
+            const urls: string[] = imgs.map(img => {
+                const srcset = (img as HTMLImageElement).getAttribute('srcset') || '';
+                if (!srcset) {
+                    return ''; // Ignore images without srcset
+                }
+
+                const parts = srcset.split(',').map(p => p.trim());
+                for (const part of parts) {
+                    const match = part.match(/^(\S+)\s+4x$/);
+                    if (match && match[1]) {
+                        return match[1]; // Found the 4x version, return it
+                    }
+                }
+
+                return ''; // No 4x version found for this image
+            }).filter(s => !!s && s.includes('pinimg')); // Filter out empty strings and non-pinterest images
+            return [...new Set(urls)]; // Return unique URLs
+        });
+
+        if (!imgs || imgs.length === 0) {
+            logger.warn(`No high-res images found on pin ${pinUrl}`);
+            return [];
+        }
+
+        const outDir = path.join(process.cwd(), 'download');
+        await fs.mkdir(outDir, { recursive: true });
+        const results: string[] = [];
+        for (let i = 0; i < imgs.length; i++) {
+            const src = imgs[i];
+            try {
+                const imgPage = await browser.newPage();
+                const resp = await imgPage.goto(src, { timeout: 30000, waitUntil: 'load' });
+                if (!resp) { await imgPage.close(); continue; }
+                const buffer = await resp.buffer();
+                const pinId = pinUrl.split('/').filter(Boolean).pop() || `pin_${Date.now()}`;
+                const timestamp = Date.now();
+                const outPath = path.join(outDir, `${pinId}_${timestamp}_${i}.png`);
+                await fs.writeFile(outPath, buffer);
+                results.push(outPath);
+                await imgPage.close();
+            } catch (err) {
+                logger.error(`Failed to download image ${src} from ${pinUrl}:`, err);
+            }
+        }
+        return results;
+    } catch (err) {
+        logger.error(`Failed to download images from ${pinUrl}:`, err);
+        return [];
+    } finally {
+        await browser.close();
+    }
+}
+
+
+// Re-usable helper to extract JSON embedded in text
+function extractJsonFromText(text: string): any | null {
+    if (!text || typeof text !== 'string') return null;
+    const fenced = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
+    if (fenced && fenced[1]) {
+        try { return JSON.parse(fenced[1].trim()); } catch (e) { /* fall through */ }
+    }
+    const brace = text.match(/\{[\s\S]*\}|\[[\s\S]*\]/);
+    if (brace && brace[0]) {
+        try { return JSON.parse(brace[0]); } catch (e) { return null; }
+    }
+    // Attempt line-separated keywords fallback
+    const lines = text.split(/\r?\n/).map((l: string) => l.trim()).filter(Boolean);
+    if (lines.length > 1) return lines;
+    return null;
+}
+
+async function extractKeywordsFromPromptWithLmstudio(prompt: string, count = 5): Promise<string[]> {
+    const instruction = `You are given a short instruction describing the type of content to search for.
+Return exactly a JSON array of ${count} short keyword phrases  suitable for searching Pinterest. `;
+
+    try {
+        const res = await callLmstudio(`${instruction}\n\nInstruction: ${prompt}`);
+        if (!res) {
+            logger.warn('callLmstudio returned empty response for keyword extraction.');
+            return [];
+        }
+
+        let parsed: any;
+        if (typeof res === 'object' && res.text) {
+            parsed = extractJsonFromText(res.text);
+        } else if (typeof res === 'string') {
+            parsed = extractJsonFromText(res);
+        } else if (typeof res === 'object') {
+            parsed = res;
+        }
+
+        if (Array.isArray(parsed)) {
+            return parsed.map(String).slice(0, count);
+        }
+
+        if (typeof parsed === 'object' && parsed !== null) {
+            const maybe = parsed.keywords || parsed.list || parsed.items || parsed.keywords_list;
+            if (Array.isArray(maybe)) return maybe.map(String).slice(0, count);
+        }
+
+        const text = typeof res === 'string' ? res : (res && res.text) || JSON.stringify(res);
+        const lines = text.split(/\r?\n/).map((l: string) => l.replace(/^\d+[\).\s-]*/, '').trim()).filter(Boolean);
+        if (lines.length >= 1) {
+            return lines.slice(0, count);
+        }
+
+        logger.warn(`Could not parse keywords from LM Studio response: ${JSON.stringify(res)}`);
+        return [];
+
+    } catch (error) {
+        logger.error('Error during keyword extraction with callLmstudio:', error);
+        return [];
+    }
+}
+
+
+(async () => {
+    logger.info(`Starting photo download process with prompt: "${HARDCODED_USER_PROMPT}"`);
+
+    // 1. Extract keywords from the hardcoded prompt
+    const keywords = await extractKeywordsFromPromptWithLmstudio(HARDCODED_USER_PROMPT, 20); // Using 5 keywords to get a good variety
+    if (!keywords || keywords.length === 0) {
+        logger.error("Could not extract keywords from prompt. Exiting.");
+        return;
+    }
+    logger.info(`Extracted keywords: ${keywords.join(', ')}`);
+
+    // 2. Search Pinterest for each keyword and collect pin URLs
+    let allPinUrls = new Set<string>();
+    for (const keyword of keywords) {
+        logger.info(`Searching Pinterest for keyword: "${keyword}"`);
+        const pinUrls = await getPinUrlsFromPinterest(keyword, SCROLL_SEARCH, PINS_TO_COLLECT);
+        pinUrls.forEach(url => allPinUrls.add(url));
+    }
+
+    const finalPinUrls = Array.from(allPinUrls);
+    logger.info(`Collected ${finalPinUrls.length} unique pin URLs to process.`);
+
+    // 3. Go through each pin URL, scroll, and download all photos
+    let totalDownloads = 0;
+    for (const pinUrl of finalPinUrls) {
+        try {
+            logger.info(`Processing pin: ${pinUrl}`);
+            const downloadedPaths = await downloadImagesFromPin(pinUrl, SCROLL_PIN);
+            if (downloadedPaths.length > 0) {
+                logger.info(`Successfully downloaded ${downloadedPaths.length} images from ${pinUrl}`);
+                totalDownloads += downloadedPaths.length;
+            } else {
+                logger.warn(`No images were downloaded from ${pinUrl}`);
+            }
+        } catch (error) {
+            logger.error(`An error occurred while processing pin ${pinUrl}:`, error);
+        }
+    }
+
+    logger.info(`Photo download process finished. Total images downloaded: ${totalDownloads}`);
+})();