ken/RandomVideoMaker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new musicspot_generator/v2 files

Browse Source
This commit is contained in:
Ken Yasue
2025-09-23 20:19:59 +02:00
parent 5d1cbf8c09
commit d74ad1b034
7 changed files with 5115 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
src/musicspot_generator/v2/generate_photo.ts Normal file
View File

@ -0,0 +1,62 @@
import fs from 'fs';
import path from 'path';
import crypto from 'crypto';
import { generateImage } from '../../lib/image-generator';
import { logger } from '../../lib/logger';
import dotenv from 'dotenv';
dotenv.config();
const scenesFilePath = path.resolve(process.cwd(), 'src/musicspot_generator/v2/scenes.json');
const GENERATED_DIR = path.resolve('generated');
const DEFAULT_SIZE = { width: 1280, height: 720 };
interface Scene {
scene: string;
imagePrompt: string;
videoPromp: string;
baseImagePath: string;
}
const COMFY_BASE_URL = process.env.COMFY_BASE_URL;
const COMFY_OUTPUT_DIR = process.env.COMFY_OUTPUT_DIR;
async function generatePhotos() {
if (!COMFY_BASE_URL || !COMFY_OUTPUT_DIR) {
throw new Error('COMFY_BASE_URL or COMFY_OUTPUT_DIR is not defined in the .env file');
}
const scenesFileContent = fs.readFileSync(scenesFilePath, 'utf-8');
const scenesData: { scenes: Scene[] } = JSON.parse(scenesFileContent);
for (const scene of scenesData.scenes) {
const hash = crypto.createHash('sha256').update(scene.baseImagePath).digest('hex');
const imgFileName = `${hash}.png`;
const outputFilePath = path.join(GENERATED_DIR, imgFileName);
if (fs.existsSync(outputFilePath)) {
logger.info(`Skipping already generated photo for: ${scene.baseImagePath}`);
continue;
}
logger.info(`Generating photo for: ${scene.baseImagePath}`);
try {
await generateImage(
scene.imagePrompt,
imgFileName,
COMFY_BASE_URL,
COMFY_OUTPUT_DIR,
'flux',
DEFAULT_SIZE
);
logger.info(`Successfully generated photo: ${imgFileName}`);
} catch (error) {
logger.error(`Error generating photo for scene ${scene.scene}:`, error);
}
}
}
generatePhotos().catch(error => {
logger.error('An unexpected error occurred:', error);
});

87
src/musicspot_generator/v2/generate_scenes.ts Normal file
View File

@ -0,0 +1,87 @@
import fs from 'fs';
import path from 'path';
import { callLMStudioAPIWithFile } from '../../lib/lmstudio';
import { logger } from '../../lib/logger';
const promptInstructions = `
Video prompt: No slowmotion, Be creative and generate dynamic dance scene.
`;
const inputDir = path.resolve(process.cwd(), 'input');
const outputFilePath = path.resolve(process.cwd(), 'src/musicspot_generator/v2/scenes.json');
interface Scene {
scene: string;
imagePrompt: string;
videoPromp: string;
baseImagePath: string;
}
async function processImages() {
const imageFiles = fs.readdirSync(inputDir).filter(file => /\.(png|jpg|jpeg)$/i.test(file));
let scenes: { scenes: Scene[] } = { scenes: [] };
if (fs.existsSync(outputFilePath)) {
const fileContent = fs.readFileSync(outputFilePath, 'utf-8');
if (fileContent) {
scenes = JSON.parse(fileContent);
}
}
for (const imageFile of imageFiles) {
const imagePath = path.resolve(inputDir, imageFile);
const absoluteImagePath = path.resolve(imagePath);
const existingScene = scenes.scenes.find(s => s.baseImagePath === absoluteImagePath);
if (existingScene) {
logger.info(`Skipping already processed image: ${imageFile}`);
continue;
}
logger.info(`Processing image: ${imageFile} `);
const prompt = `
Analyze the provided image and generate a JSON object with the following structure:
{
"scenes": [
{
"scene": "A descriptive title for the scene in the image.",
"imagePrompt": {
"description": "A detailed description of the image content.",
"style": "Art style or photography style of the image.",
"lighting": "Description of the lighting in the image.",
"outfit": "Description of the outfit or clothing style in the image.",
"location": "Description of the location or setting of the image.",
"poses": "Description of the poses or actions of any subjects in the image.",
"angle": "Description of the camera angle or perspective of the image.",
}
"videoPromp": "Based on the image, create a prompt for a video that shows what might happen next or brings the scene to life.",
"baseImagePath": "The absolute path of the base image."
}
]
}
Instructions: ${promptInstructions}
`;
try {
const result = await callLMStudioAPIWithFile(imagePath, prompt);
if (result && result.scenes) {
const newScene = result.scenes[0];
newScene.baseImagePath = absoluteImagePath; // Ensure the path is correct
scenes.scenes.push(newScene);
fs.writeFileSync(outputFilePath, JSON.stringify(scenes, null, 2));
logger.info(`Successfully processed and saved scene for: ${imageFile} `);
} else {
logger.error('Failed to get valid scene data from API for image:', imageFile);
}
} catch (error) {
logger.error(`Error processing image ${imageFile}: `, error);
}
}
}
processImages().catch(error => {
logger.error('An unexpected error occurred:', error);
});

57
src/musicspot_generator/v2/generate_video.ts Normal file
View File

@ -0,0 +1,57 @@
import * as fs from 'fs';
import * as path from 'path';
import * as crypto from 'crypto';
import { generateVideo } from '../../lib/video-generator';
import dotenv from 'dotenv';
dotenv.config();
interface Scene {
scene: string;
videoPrompt: string;
baseImagePath: string;
}
const scenesFilePath = path.join(__dirname, 'scenes.json');
const generatedFolderPath = path.join(__dirname, '..', '..', '..', 'generated');
async function processScenes() {
try {
const scenesData = fs.readFileSync(scenesFilePath, 'utf-8');
const scenes: Scene[] = JSON.parse(scenesData).scenes;
for (const scene of scenes) {
const hash = crypto.createHash('sha256').update(scene.baseImagePath).digest('hex');
const imageFileName = `${hash}.png`;
const imagePath = path.join(generatedFolderPath, imageFileName);
if (fs.existsSync(imagePath)) {
const outputVideoFileName = `${hash}.mp4`;
const outputVideoPath = path.join(generatedFolderPath, outputVideoFileName);
if (fs.existsSync(outputVideoPath)) {
console.log(`Video already exists for scene ${scene.scene}, skipping.`);
continue;
}
console.log(`Generating video for scene ${scene.scene}...`);
await generateVideo(
scene.videoPrompt,
imagePath,
outputVideoPath,
process.env.COMFY_BASE_URL!,
process.env.COMFY_OUTPUT_DIR!,
{ width: 1280, height: 720 }
);
console.log(`Video for scene ${scene.scene} saved to ${outputVideoPath}`);
} else {
console.warn(`Image not found for scene ${scene.scene}: ${imagePath}`);
}
}
} catch (error) {
console.error('Error processing scenes:', error);
}
}
processScenes();

227
src/musicspot_generator/v2/photo_download.ts Normal file
View File

@ -0,0 +1,227 @@
import { callLmstudio } from '../../lib/lmstudio';
import { logger } from '../../lib/logger';
import * as fs from 'fs/promises';
import dotenv from 'dotenv';
import path from 'path';
import puppeteer from 'puppeteer';
dotenv.config();
const SCROLL_SEARCH = 3; // scroll times on search results
const SCROLL_PIN = 3; // scroll times on pin page
const PINS_TO_COLLECT = 5;
// Hard-coded user prompt
const HARDCODED_USER_PROMPT = process.env.HARDCODED_USER_PROMPT || `
Generate 20 keywords for photos of group of people dancing together focus on street and urban style. All keywords shoudld contain \"group horizontal\" and what you create.
Example output : ["group horizontal hiphop dance","group horizontal modern dance","",... and 20 items in array]
`;
async function getPinUrlsFromPinterest(keyword: string, scrollCount = SCROLL_SEARCH, limit = PINS_TO_COLLECT): Promise<string[]> {
const browser = await puppeteer.launch({ headless: true });
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36');
await page.setViewport({ width: 1920, height: 1080 });
try {
const searchUrl = `https://www.pinterest.com/search/pins/?q=${encodeURIComponent(keyword)}`;
await page.goto(searchUrl, { waitUntil: 'networkidle2' });
let pinLinks = new Set<string>();
for (let i = 0; i < scrollCount; i++) {
const linksBefore = pinLinks.size;
const newLinks = await page.$$eval('a', (anchors) =>
anchors.map((a) => a.href).filter((href) => href.includes('/pin/'))
);
newLinks.forEach(link => pinLinks.add(link));
if (pinLinks.size >= limit) {
break;
}
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
await new Promise(r => setTimeout(r, 500 + Math.random() * 1000));
if (pinLinks.size === linksBefore) {
// If no new pins are loaded, stop scrolling
logger.info(`No new pins loaded for "${keyword}", stopping scroll.`);
break;
}
}
return Array.from(pinLinks).slice(0, limit);
} catch (error) {
logger.error(`Error while getting pin URLs from Pinterest for keyword "${keyword}":`, error);
return [];
} finally {
await browser.close();
}
}
async function downloadImagesFromPin(pinUrl: string, scrollTimes = SCROLL_PIN): Promise<string[]> {
const browser = await puppeteer.launch({ headless: true });
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36');
await page.setViewport({ width: 1920, height: 1080 });
try {
await page.goto(pinUrl, { waitUntil: 'networkidle2', timeout: 30000 });
for (let i = 0; i < scrollTimes; i++) {
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
await new Promise((r) => setTimeout(r, 700 + Math.random() * 800));
}
const imgs: string[] = await page.$$eval('img', imgs => {
const urls: string[] = imgs.map(img => {
const srcset = (img as HTMLImageElement).getAttribute('srcset') || '';
if (!srcset) {
return ''; // Ignore images without srcset
}
const parts = srcset.split(',').map(p => p.trim());
for (const part of parts) {
const match = part.match(/^(\S+)\s+4x$/);
if (match && match[1]) {
return match[1]; // Found the 4x version, return it
}
}
return ''; // No 4x version found for this image
}).filter(s => !!s && s.includes('pinimg')); // Filter out empty strings and non-pinterest images
return [...new Set(urls)]; // Return unique URLs
});
if (!imgs || imgs.length === 0) {
logger.warn(`No high-res images found on pin ${pinUrl}`);
return [];
}
const outDir = path.join(process.cwd(), 'download');
await fs.mkdir(outDir, { recursive: true });
const results: string[] = [];
for (let i = 0; i < imgs.length; i++) {
const src = imgs[i];
try {
const imgPage = await browser.newPage();
const resp = await imgPage.goto(src, { timeout: 30000, waitUntil: 'load' });
if (!resp) { await imgPage.close(); continue; }
const buffer = await resp.buffer();
const pinId = pinUrl.split('/').filter(Boolean).pop() || `pin_${Date.now()}`;
const timestamp = Date.now();
const outPath = path.join(outDir, `${pinId}_${timestamp}_${i}.png`);
await fs.writeFile(outPath, buffer);
results.push(outPath);
await imgPage.close();
} catch (err) {
logger.error(`Failed to download image ${src} from ${pinUrl}:`, err);
}
}
return results;
} catch (err) {
logger.error(`Failed to download images from ${pinUrl}:`, err);
return [];
} finally {
await browser.close();
}
}
// Re-usable helper to extract JSON embedded in text
function extractJsonFromText(text: string): any | null {
if (!text || typeof text !== 'string') return null;
const fenced = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
if (fenced && fenced[1]) {
try { return JSON.parse(fenced[1].trim()); } catch (e) { /* fall through */ }
}
const brace = text.match(/\{[\s\S]*\}|\[[\s\S]*\]/);
if (brace && brace[0]) {
try { return JSON.parse(brace[0]); } catch (e) { return null; }
}
// Attempt line-separated keywords fallback
const lines = text.split(/\r?\n/).map((l: string) => l.trim()).filter(Boolean);
if (lines.length > 1) return lines;
return null;
}
async function extractKeywordsFromPromptWithLmstudio(prompt: string, count = 5): Promise<string[]> {
const instruction = `You are given a short instruction describing the type of content to search for.
Return exactly a JSON array of ${count} short keyword phrases suitable for searching Pinterest. `;
try {
const res = await callLmstudio(`${instruction}\n\nInstruction: ${prompt}`);
if (!res) {
logger.warn('callLmstudio returned empty response for keyword extraction.');
return [];
}
let parsed: any;
if (typeof res === 'object' && res.text) {
parsed = extractJsonFromText(res.text);
} else if (typeof res === 'string') {
parsed = extractJsonFromText(res);
} else if (typeof res === 'object') {
parsed = res;
}
if (Array.isArray(parsed)) {
return parsed.map(String).slice(0, count);
}
if (typeof parsed === 'object' && parsed !== null) {
const maybe = parsed.keywords || parsed.list || parsed.items || parsed.keywords_list;
if (Array.isArray(maybe)) return maybe.map(String).slice(0, count);
}
const text = typeof res === 'string' ? res : (res && res.text) || JSON.stringify(res);
const lines = text.split(/\r?\n/).map((l: string) => l.replace(/^\d+[\).\s-]*/, '').trim()).filter(Boolean);
if (lines.length >= 1) {
return lines.slice(0, count);
}
logger.warn(`Could not parse keywords from LM Studio response: ${JSON.stringify(res)}`);
return [];
} catch (error) {
logger.error('Error during keyword extraction with callLmstudio:', error);
return [];
}
}
(async () => {
logger.info(`Starting photo download process with prompt: "${HARDCODED_USER_PROMPT}"`);
// 1. Extract keywords from the hardcoded prompt
const keywords = await extractKeywordsFromPromptWithLmstudio(HARDCODED_USER_PROMPT, 20); // Using 5 keywords to get a good variety
if (!keywords || keywords.length === 0) {
logger.error("Could not extract keywords from prompt. Exiting.");
return;
}
logger.info(`Extracted keywords: ${keywords.join(', ')}`);
// 2. Search Pinterest for each keyword and collect pin URLs
let allPinUrls = new Set<string>();
for (const keyword of keywords) {
logger.info(`Searching Pinterest for keyword: "${keyword}"`);
const pinUrls = await getPinUrlsFromPinterest(keyword, SCROLL_SEARCH, PINS_TO_COLLECT);
pinUrls.forEach(url => allPinUrls.add(url));
}
const finalPinUrls = Array.from(allPinUrls);
logger.info(`Collected ${finalPinUrls.length} unique pin URLs to process.`);
// 3. Go through each pin URL, scroll, and download all photos
let totalDownloads = 0;
for (const pinUrl of finalPinUrls) {
try {
logger.info(`Processing pin: ${pinUrl}`);
const downloadedPaths = await downloadImagesFromPin(pinUrl, SCROLL_PIN);
if (downloadedPaths.length > 0) {
logger.info(`Successfully downloaded ${downloadedPaths.length} images from ${pinUrl}`);
totalDownloads += downloadedPaths.length;
} else {
logger.warn(`No images were downloaded from ${pinUrl}`);
}
} catch (error) {
logger.error(`An error occurred while processing pin ${pinUrl}:`, error);
}
}
logger.info(`Photo download process finished. Total images downloaded: ${totalDownloads}`);
})();

4652
src/musicspot_generator/v2/scenes.json Normal file
View File

File diff suppressed because it is too large Load Diff