save changes

src/musicspot_generator/v2/generate_photo.ts

@@ -1,19 +1,28 @@
 import fs from 'fs';
 import path from 'path';
 import crypto from 'crypto';
-import { generateImage } from '../../lib/image-generator';
+import { generateImage } from '../../lib/image-generator-style-faceswap';
 import { logger } from '../../lib/logger';
 import dotenv from 'dotenv';
 
 dotenv.config();
 
 const scenesFilePath = path.resolve(process.cwd(), 'src/musicspot_generator/v2/scenes.json');
+const faceFilePath = path.resolve(process.cwd(), 'src/musicspot_generator/v2/face.png');
 const GENERATED_DIR = path.resolve('generated');
 const DEFAULT_SIZE = { width: 1280, height: 720 };
 
 interface Scene {
     scene: string;
-    imagePrompt: string;
+    imagePrompt: {
+        description: string,
+        style: string,
+        lighting: string,
+        outfit: string,
+        location: string,
+        poses: string,
+        angle: string,
+    };
     videoPromp: string;
     baseImagePath: string;
 }
@@ -43,11 +52,12 @@ async function generatePhotos() {
 
         try {
             await generateImage(
-                scene.imagePrompt,
+                `realistic photo of woman, wavy long brown hair, fullbody shot, ${scene.imagePrompt.location},${scene.imagePrompt.angle},${scene.imagePrompt.lighting},${scene.imagePrompt.outfit}`,
+                faceFilePath,
+                scene.baseImagePath,
                 imgFileName,
                 COMFY_BASE_URL,
                 COMFY_OUTPUT_DIR,
-                'flux',
                 DEFAULT_SIZE
             );
             logger.info(`Successfully generated photo: ${imgFileName}`);

src/musicspot_generator/v2/generate_scenes.ts

@@ -4,9 +4,9 @@ import { callLMStudioAPIWithFile } from '../../lib/lmstudio';
 import { logger } from '../../lib/logger';
 
 const promptInstructions = `
-    Video prompt: No slowmotion, Be creative and generate dynamic dance scene.
+    Video prompt: No slowmotion, Be creative and generate gengle action scene.
     `;
-
+6
 const inputDir = path.resolve(process.cwd(), 'input');
 const outputFilePath = path.resolve(process.cwd(), 'src/musicspot_generator/v2/scenes.json');
 

src/musicspot_generator/v2/generate_video.ts

@@ -21,33 +21,39 @@ async function processScenes() {
         const scenes: Scene[] = JSON.parse(scenesData).scenes;
 
         for (const scene of scenes) {
-            const hash = crypto.createHash('sha256').update(scene.baseImagePath).digest('hex');
-            const imageFileName = `${hash}.png`;
-            const imagePath = path.join(generatedFolderPath, imageFileName);
 
-            if (fs.existsSync(imagePath)) {
-                const outputVideoFileName = `${hash}.mp4`;
-                const outputVideoPath = path.join(generatedFolderPath, outputVideoFileName);
+            try {
+                const hash = crypto.createHash('sha256').update(scene.baseImagePath).digest('hex');
+                const imageFileName = `${hash}.png`;
+                const imagePath = path.join(generatedFolderPath, imageFileName);
 
-                if (fs.existsSync(outputVideoPath)) {
-                    console.log(`Video already exists for scene ${scene.scene}, skipping.`);
-                    continue;
+                if (fs.existsSync(imagePath)) {
+                    const outputVideoFileName = `${hash}.mp4`;
+                    const outputVideoPath = path.join(generatedFolderPath, outputVideoFileName);
+
+                    if (fs.existsSync(outputVideoPath)) {
+                        console.log(`Video already exists for scene ${scene.scene}, skipping.`);
+                        continue;
+                    }
+
+                    console.log(`Generating video for scene ${scene.scene}...`);
+
+                    await generateVideo(
+                        scene.videoPrompt,
+                        imagePath,
+                        outputVideoPath,
+                        process.env.COMFY_BASE_URL!,
+                        process.env.COMFY_OUTPUT_DIR!,
+                        { width: 1280, height: 720 }
+                    );
+                    console.log(`Video for scene ${scene.scene} saved to ${outputVideoPath}`);
+                } else {
+                    console.warn(`Image not found for scene ${scene.scene}: ${imagePath}`);
                 }
-
-                console.log(`Generating video for scene ${scene.scene}...`);
-
-                await generateVideo(
-                    scene.videoPrompt,
-                    imagePath,
-                    outputVideoPath,
-                    process.env.COMFY_BASE_URL!,
-                    process.env.COMFY_OUTPUT_DIR!,
-                    { width: 1280, height: 720 }
-                );
-                console.log(`Video for scene ${scene.scene} saved to ${outputVideoPath}`);
-            } else {
-                console.warn(`Image not found for scene ${scene.scene}: ${imagePath}`);
+            } catch (e) {
+                continue;
             }
+
         }
     } catch (error) {
         console.error('Error processing scenes:', error);

src/musicspot_generator/v2/generate_video_from_static_image.ts

@@ -0,0 +1,68 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+import { generateVideo } from '../../lib/video-generator';
+import { callLMStudioAPIWithFile } from '../../lib/lmstudio';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+const inputFolderPath = path.join(__dirname, '..', '..', '..', 'input');
+const generatedFolderPath = path.join(__dirname, '..', '..', '..', 'generated');
+
+async function processImages() {
+    try {
+        const imageFiles = fs.readdirSync(inputFolderPath);
+
+        for (const imageFile of imageFiles) {
+            const imagePath = path.join(inputFolderPath, imageFile);
+
+            try {
+                const hash = crypto.createHash('sha256').update(imageFile).digest('hex');
+                const outputVideoFileName = `${hash}.mp4`;
+                const outputVideoPath = path.join(generatedFolderPath, outputVideoFileName);
+
+                if (fs.existsSync(outputVideoPath)) {
+                    console.log(`Video already exists for image ${imageFile}, skipping.`);
+                    continue;
+                }
+
+                console.log(`Generating video prompt for image ${imageFile}...`);
+                const promptResult = await callLMStudioAPIWithFile(
+                    imagePath,
+                    `
+                    Generate a short, dancing video prompt for an image located at ${imagePath}. 
+                    Return the result in this format: {"result":""}
+                    Instruction:
+                    - Find best dancing expression based on the photo
+                `);
+                const videoPrompt = promptResult.result;
+                console.log(`Video prompt ${videoPrompt}`);
+
+                if (!videoPrompt) {
+                    console.error(`Could not generate video prompt for image ${imageFile}`);
+                    continue;
+                }
+
+                console.log(`Generating video for image ${imageFile}...`);
+
+                await generateVideo(
+                    videoPrompt,
+                    imagePath,
+                    outputVideoPath,
+                    process.env.COMFY_BASE_URL!,
+                    process.env.COMFY_OUTPUT_DIR!,
+                    { width: 1280, height: 720 }
+                );
+                console.log(`Video for image ${imageFile} saved to ${outputVideoPath}`);
+            } catch (e) {
+                console.error(`Error processing image ${imageFile}:`, e);
+                continue;
+            }
+        }
+    } catch (error) {
+        console.error('Error processing images:', error);
+    }
+}
+
+processImages();

src/musicspot_generator/v2/photo_download.ts

@@ -13,8 +13,8 @@ const PINS_TO_COLLECT = 5;
 
 // Hard-coded user prompt
 const HARDCODED_USER_PROMPT = process.env.HARDCODED_USER_PROMPT || `
-    Generate 20 keywords for photos of group of people dancing together focus on street and urban style. All keywords shoudld contain \"group horizontal\" and what you create.
-    Example output : ["group horizontal hiphop dance","group horizontal modern dance","",... and 20 items in array]
+    Generate 20 keywords for photos of a girl in scary scene for music video for haloween. All keywords should containe girl in a scene.
+    Example output : ["a girl in grave yard,"a girl in scary forest","",... and 20 items in array]
     `;
 
 async function getPinUrlsFromPinterest(keyword: string, scrollCount = SCROLL_SEARCH, limit = PINS_TO_COLLECT): Promise<string[]> {