ken/RandomVideoMaker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

save changes

Browse Source
This commit is contained in:
Ken Yasue
2025-08-27 12:07:51 +02:00
parent d4028e8f7f
commit 153e50a356
3 changed files with 462 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
src/comfyworkflows/generate_image_mix_style.json Normal file
View File

@ -0,0 +1,240 @@
{
"1": {
"inputs": {
"clip_name1": "clip_l.safetensors",
"clip_name2": "t5xxl_fp16.safetensors",
"type": "flux",
"device": "default"
},
"class_type": "DualCLIPLoader",
"_meta": {
"title": "DualCLIPLoader"
}
},
"2": {
"inputs": {
"unet_name": "flux1-krea-dev_fp8_scaled.safetensors",
"weight_dtype": "default"
},
"class_type": "UNETLoader",
"_meta": {
"title": "Load Diffusion Model"
}
},
"3": {
"inputs": {
"width": 320,
"height": 640,
"batch_size": 1
},
"class_type": "EmptySD3LatentImage",
"_meta": {
"title": "EmptySD3LatentImage"
}
},
"4": {
"inputs": {
"seed": 803508963683741,
"steps": 20,
"cfg": 1,
"sampler_name": "euler",
"scheduler": "simple",
"denoise": 1,
"model": [
"2",
0
],
"positive": [
"18",
0
],
"negative": [
"5",
0
],
"latent_image": [
"3",
0
]
},
"class_type": "KSampler",
"_meta": {
"title": "KSampler"
}
},
"5": {
"inputs": {
"conditioning": [
"14",
0
]
},
"class_type": "ConditioningZeroOut",
"_meta": {
"title": "ConditioningZeroOut"
}
},
"6": {
"inputs": {
"samples": [
"4",
0
],
"vae": [
"12",
0
]
},
"class_type": "VAEDecode",
"_meta": {
"title": "VAE Decode"
}
},
"8": {
"inputs": {
"style_model_name": "flux1-redux-dev.safetensors"
},
"class_type": "StyleModelLoader",
"_meta": {
"title": "Load Style Model"
}
},
"9": {
"inputs": {
"crop": "center",
"clip_vision": [
"10",
0
],
"image": [
"13",
0
]
},
"class_type": "CLIPVisionEncode",
"_meta": {
"title": "CLIP Vision Encode"
}
},
"10": {
"inputs": {
"clip_name": "sigclip_vision_patch14_384.safetensors"
},
"class_type": "CLIPVisionLoader",
"_meta": {
"title": "Load CLIP Vision"
}
},
"11": {
"inputs": {
"strength": 0.20000000000000004,
"conditioning": [
"14",
0
],
"style_model": [
"8",
0
],
"clip_vision_output": [
"9",
0
]
},
"class_type": "ApplyStyleModelAdjust",
"_meta": {
"title": "Apply Style Model (Adjusted)"
}
},
"12": {
"inputs": {
"vae_name": "ae.safetensors"
},
"class_type": "VAELoader",
"_meta": {
"title": "Load VAE"
}
},
"13": {
"inputs": {
"image": "fd2dc3bbc879703b03abf892d4189667.jpg"
},
"class_type": "LoadImage",
"_meta": {
"title": "Load Image 1"
}
},
"14": {
"inputs": {
"text": "Ethereal realistic girl with flowing blue hair, glowing sparkles and stardust, elegant backless dress shimmering with cosmic light, dreamy profile pose, soft glowing skin, fantasy night atmosphere, luminous and magical aesthetic",
"clip": [
"1",
0
]
},
"class_type": "CLIPTextEncode",
"_meta": {
"title": "CLIP Text Encode (Prompt)"
}
},
"15": {
"inputs": {
"filename_prefix": "STYLEDVIDEOMAKER",
"images": [
"6",
0
]
},
"class_type": "SaveImage",
"_meta": {
"title": "Save Image"
}
},
"16": {
"inputs": {
"image": "a09ee3d44fff05f20c88976555f8fa10.jpg"
},
"class_type": "LoadImage",
"_meta": {
"title": "Load Image 2"
}
},
"17": {
"inputs": {
"crop": "center",
"clip_vision": [
"10",
0
],
"image": [
"16",
0
]
},
"class_type": "CLIPVisionEncode",
"_meta": {
"title": "CLIP Vision Encode"
}
},
"18": {
"inputs": {
"strength": 0.20000000000000004,
"conditioning": [
"11",
0
],
"style_model": [
"8",
0
],
"clip_vision_output": [
"17",
0
]
},
"class_type": "ApplyStyleModelAdjust",
"_meta": {
"title": "Apply Style Model (Adjusted)"
}
}
}

78
src/lib/image-generator-mix-style.ts Normal file
View File

@ -0,0 +1,78 @@
import * as fs from 'fs/promises';
import * as path from 'path';
import axios from 'axios';
import dotenv from 'dotenv';
dotenv.config();
interface ImageSize {
width: number;
height: number;
}
async function generateImage(
prompt: string,
imageName1: string,
imageName2: string,
newFileName: string,
comfyBaseUrl: string,
comfyOutputDir: string,
size: ImageSize = { width: 720, height: 1280 }
): Promise<string> {
const COMFY_BASE_URL = comfyBaseUrl.replace(/\/$/, '');
const COMFY_OUTPUT_DIR = comfyOutputDir;
let workflow;
workflow = JSON.parse(await fs.readFile('src/comfyworkflows/generate_image_mix_style.json', 'utf-8'));
workflow['14']['inputs']['text'] = prompt;
// Set image name
workflow['13']['inputs']['image'] = imageName1;
// Set image name
workflow['16']['inputs']['image'] = imageName2;
workflow['3']['inputs']['width'] = size.width;
workflow['3']['inputs']['height'] = size.height;
const response = await axios.post(`${COMFY_BASE_URL}/prompt`, { prompt: workflow });
const promptId = response.data.prompt_id;
let history;
do {
await new Promise(resolve => setTimeout(resolve, 1000));
const historyResponse = await axios.get(`${COMFY_BASE_URL}/history/${promptId}`);
history = historyResponse.data[promptId];
} while (!history || Object.keys(history.outputs).length === 0);
const files = await fs.readdir(COMFY_OUTPUT_DIR!);
const generatedFiles = files.filter(file => file.startsWith('STYLEDVIDEOMAKER'));
const fileStats = await Promise.all(
generatedFiles.map(async (file) => {
const stat = await fs.stat(path.join(COMFY_OUTPUT_DIR!, file));
return { file, mtime: stat.mtime };
})
);
fileStats.sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
const latestFile = fileStats[0].file;
const newFilePath = path.resolve('./generated', newFileName);
await fs.mkdir('./generated', { recursive: true });
const sourcePath = path.join(COMFY_OUTPUT_DIR!, latestFile);
try {
await fs.unlink(newFilePath);
} catch (err) {
// ignore if not exists
}
await fs.copyFile(sourcePath, newFilePath);
return newFilePath;
}
export { generateImage };

228
src/piterest_styletransfer_video.ts
View File

@ -1,6 +1,7 @@
import { downloadImagesFromPinterestPin } from './lib/downloader';
import { callOpenAIWithFile } from './lib/openai';
import { generateStyledVideo } from './lib/video-generator-styled';
import { generateVideo } from './lib/video-generator';
import { generateImage } from './lib/image-generator-mix-style';
import { logger } from './lib/logger';
import * as fs from 'fs/promises';
import dotenv from 'dotenv';
@ -28,7 +29,8 @@ interface GenerationTask {
imagePrompt: string;
videoPrompt: string;
imageFileName: string;
renamedImagePath: string;
renamedImagePaths: string[];
generatedImagePath?: string;
genre: string;
subGenre: string;
scene: string;
@ -36,17 +38,23 @@ interface GenerationTask {
camera: string;
}
async function getPromptsForImage(imagePath: string, pinUrl: string, genre: string, subGenre: string): Promise<GenerationTask | null> {
async function getPromptsForImage(imagePaths: string[], pinUrl: string, genre: string, subGenre: string): Promise<GenerationTask | null> {
const pinId = pinUrl.split('/').filter(Boolean).pop() || `pin_${Date.now()}`;
const timestamp = new Date().getTime();
const imageFileName = `${pinId}_${timestamp}.png`;
const renamedImagePath = path.join(path.dirname(imagePath), imageFileName);
const renamedImagePaths = [];
for (let i = 0; i < imagePaths.length; i++) {
const renamedPath = path.join(path.dirname(imagePaths[i]), `${pinId}_${timestamp}_${i}.png`);
await fs.rename(imagePaths[i], renamedPath);
renamedImagePaths.push(renamedPath);
}
logger.debug(`Renamed source images to: ${renamedImagePaths.join(', ')}`);
const imageForPrompt = renamedImagePaths[Math.floor(Math.random() * renamedImagePaths.length)];
try {
await fs.rename(imagePath, renamedImagePath);
logger.debug(`Renamed ${imagePath} to ${renamedImagePath}`);
const promptResponse = await callOpenAIWithFile(renamedImagePath,
const promptResponse = await callOpenAIWithFile(imageForPrompt,
`Analyze the provided image and generate the following:
1. 'scene': A description of the image's environment.
2. 'action': A description of the main action occurring in the image.
@ -66,101 +74,71 @@ async function getPromptsForImage(imagePath: string, pinUrl: string, genre: stri
---
`);
const { scene, action, camera, image_prompt: imagePrompt, video_prompt: videoPrompt } = promptResponse;
logger.info(`Image prompt for ${renamedImagePath}:`, imagePrompt);
logger.info(`Video prompt for ${renamedImagePath}:`, videoPrompt);
logger.info(`Image prompt for ${imageForPrompt}:`, imagePrompt);
logger.info(`Video prompt for ${imageForPrompt}:`, videoPrompt);
return { pinUrl, imagePrompt, videoPrompt, imageFileName, renamedImagePath, genre, subGenre, scene, action, camera };
return { pinUrl, imagePrompt, videoPrompt, imageFileName, renamedImagePaths, genre, subGenre, scene, action, camera };
} catch (error) {
logger.error(`Failed to get prompts for ${renamedImagePath}:`, error);
logger.error(`Failed to get prompts for ${imageForPrompt}:`, error);
for (const p of renamedImagePaths) {
try {
await fs.unlink(renamedImagePath);
await fs.unlink(p);
} catch (cleanupError) {
// ignore
}
}
return null;
}
}
async function generateImageAndVideo(task: GenerationTask, server: { baseUrl: string; outputDir: string; }): Promise<{ imagePath: string; videoPath: string; } | null> {
const { imagePrompt, videoPrompt, imageFileName, renamedImagePath } = task;
async function generateImageForTask(task: GenerationTask, server: { baseUrl: string; outputDir: string; }): Promise<string | null> {
const { imagePrompt, imageFileName, renamedImagePaths } = task;
const { baseUrl, outputDir } = server;
const inputDir = outputDir.replace("output", "input");
const sourceFileNames: string[] = [];
try {
const destPath = path.join(inputDir, imageFileName);
await fs.copyFile(renamedImagePath, destPath);
logger.info(`Copied ${renamedImagePath} to ${destPath}`);
for (const sourcePath of renamedImagePaths) {
const fileName = path.basename(sourcePath);
const destPath = path.join(inputDir, fileName);
await fs.copyFile(sourcePath, destPath);
sourceFileNames.push(fileName);
logger.info(`Copied ${sourcePath} to ${destPath}`);
}
const videoFileName = imageFileName.replace('.png', '.mp4');
const { videoPath, imagePath } = await generateStyledVideo(
const generatedImagePath = await generateImage(
imagePrompt,
videoPrompt,
sourceFileNames[0],
sourceFileNames[1],
imageFileName,
videoFileName,
baseUrl,
outputDir,
{ width: 720, height: 1280 }
);
path.join(outputDir, videoFileName);
return { imagePath: imagePath, videoPath };
return generatedImagePath;
} catch (error) {
logger.error(`Failed to generate styled video for ${imageFileName} on server ${baseUrl}:`, error);
logger.error(`Failed to generate image for ${imageFileName} on server ${baseUrl}:`, error);
return null;
} finally {
for (const sourcePath of renamedImagePaths) {
try {
await fs.unlink(renamedImagePath);
logger.debug(`Deleted renamed source image: ${renamedImagePath}`);
await fs.unlink(sourcePath);
logger.debug(`Deleted source image: ${sourcePath}`);
} catch (error) {
logger.error(`Failed to delete renamed source image ${renamedImagePath}:`, error);
logger.error(`Failed to delete source image ${sourcePath}:`, error);
}
}
}
async function worker(id: number, server: { baseUrl: string; outputDir: string; }, taskQueue: GenerationTask[]) {
logger.info(`Worker ${id} started for server ${server.baseUrl}`);
while (taskQueue.length > 0) {
const task = taskQueue.shift();
if (task) {
logger.info(`Worker ${id} processing task: ${task.imageFileName}`);
const result = await generateImageAndVideo(task, server);
if (result) {
for (const fileName of sourceFileNames) {
try {
const videoData = {
genre: task.genre,
sub_genre: task.subGenre,
scene: task.scene,
action: task.action,
camera: task.camera,
image_prompt: task.imagePrompt,
video_prompt: task.videoPrompt,
image_path: result.imagePath,
video_path: result.videoPath,
};
const videoId = await VideoModel.create(videoData);
logger.info(`Successfully saved video record to database with ID: ${videoId}`);
const newImageName = `${videoId}_${task.genre}_${task.subGenre}${path.extname(result.imagePath)}`;
const newVideoName = `${videoId}_${task.genre}_${task.subGenre}${path.extname(result.videoPath)}`;
const newImagePath = path.join(path.dirname(result.imagePath), newImageName);
const newVideoPath = path.join(path.dirname(result.videoPath), newVideoName);
await fs.rename(result.imagePath, newImagePath);
await fs.rename(result.videoPath, newVideoPath);
await VideoModel.update(videoId, {
image_path: newImagePath,
video_path: newVideoPath,
});
logger.info(`Renamed files and updated database record for video ID: ${videoId}`);
const serverPath = path.join(inputDir, fileName);
await fs.unlink(serverPath);
logger.debug(`Deleted server image: ${serverPath}`);
} catch (error) {
logger.error('Failed to save video record to database or rename files:', error);
logger.error(`Failed to delete server image ${fileName}:`, error);
}
}
}
}
logger.info(`Worker ${id} finished.`);
}
async function getPinUrlFromPinterest(keyword: string): Promise<string | null> {
const browser = await puppeteer.launch({ headless: true });
@ -212,6 +190,7 @@ async function getPinUrlFromPinterest(keyword: string): Promise<string | null> {
}
while (true) {
const generationTasks: GenerationTask[] = [];
for (const genreSubGenre of keywords) {
@ -235,8 +214,6 @@ async function getPinUrlFromPinterest(keyword: string): Promise<string | null> {
continue;
}
const generationTasks: GenerationTask[] = [];
logger.info(`--- Starting processing for pin: ${pin} ---`);
const downloadedImagePaths = await downloadImagesFromPinterestPin(pin);
if (downloadedImagePaths.length === 0) {
@ -244,36 +221,107 @@ async function getPinUrlFromPinterest(keyword: string): Promise<string | null> {
continue;
}
const selectedImages = downloadedImagePaths.sort(() => 0.5 - Math.random()).slice(0, 1);
const selectedImages = downloadedImagePaths.sort(() => 0.5 - Math.random()).slice(0, 2);
logger.info(`--- Randomly selected ${selectedImages.length} images for processing ---`);
for (const imagePath of selectedImages) {
const pinId = path.basename(imagePath, path.extname(imagePath)).split('_related_')[0];
if (selectedImages.length === 2) {
const pinId = path.basename(selectedImages[0], path.extname(selectedImages[0])).split('_related_')[0];
const pinUrl = `https://www.pinterest.com/pin/${pinId}/`;
const task = await getPromptsForImage(imagePath, pinUrl, genre, subGenre);
const task = await getPromptsForImage(selectedImages, pinUrl, genre, subGenre);
if (task) {
generationTasks.push(task);
}
}
const unselectedImages = downloadedImagePaths.filter(p => !selectedImages.includes(p));
for (const imagePath of unselectedImages) {
try {
} else {
logger.warn(`Skipping pin ${pin} as it did not have at least 2 images.`);
for (const imagePath of selectedImages) {
await fs.unlink(imagePath);
logger.debug(`Deleted unselected image: ${imagePath}`);
}
}
}
// --- Image Generation Phase ---
logger.info(`--- Starting image generation for ${generationTasks.length} tasks ---`);
for (const task of generationTasks) {
const server = servers[Math.floor(Math.random() * servers.length)];
const imagePath = await generateImageForTask(task, server);
if (imagePath) {
task.generatedImagePath = imagePath;
}
}
logger.info("--- Finished image generation ---");
// --- Video Generation Phase ---
logger.info(`--- Starting video generation for ${generationTasks.length} tasks ---`);
for (const task of generationTasks) {
if (!task.generatedImagePath) {
logger.warn(`Skipping video generation for task ${task.imageFileName} as it has no generated image.`);
continue;
}
const server = servers[Math.floor(Math.random() * servers.length)];
const inputDir = server.outputDir.replace("output", "input");
const generatedImageName = path.basename(task.generatedImagePath);
const serverImagePath = path.join(inputDir, generatedImageName);
try {
await fs.copyFile(task.generatedImagePath, serverImagePath);
logger.info(`Copied ${task.generatedImagePath} to ${serverImagePath}`);
const videoFileName = task.imageFileName.replace('.png', '.mp4');
const videoPath = await generateVideo(
task.videoPrompt,
generatedImageName,
videoFileName,
server.baseUrl,
server.outputDir,
{ width: 720, height: 1280 }
);
if (videoPath) {
const videoData = {
genre: task.genre,
sub_genre: task.subGenre,
scene: task.scene,
action: task.action,
camera: task.camera,
image_prompt: task.imagePrompt,
video_prompt: task.videoPrompt,
image_path: task.generatedImagePath,
video_path: videoPath,
};
const videoId = await VideoModel.create(videoData);
logger.info(`Successfully saved video record to database with ID: ${videoId}`);
const newImageName = `${videoId}_${task.genre}_${task.subGenre}${path.extname(task.generatedImagePath)}`;
const newVideoName = `${videoId}_${task.genre}_${task.subGenre}${path.extname(videoPath)}`;
const newImagePath = path.join(path.dirname(task.generatedImagePath), newImageName);
const newVideoPath = path.join(path.dirname(videoPath), newVideoName);
await fs.rename(task.generatedImagePath, newImagePath);
await fs.rename(videoPath, newVideoPath);
await VideoModel.update(videoId, {
image_path: newImagePath,
video_path: newVideoPath,
});
logger.info(`Renamed files and updated database record for video ID: ${videoId}`);
await fs.unlink(newImagePath);
logger.info(`Deleted paired image: ${newImagePath}`);
}
} catch (error) {
logger.error(`Failed to delete unselected image ${imagePath}:`, error);
logger.error('An error occurred during video generation or database operations:', error);
} finally {
try {
await fs.unlink(serverImagePath);
logger.debug(`Deleted server image: ${serverImagePath}`);
} catch (error) {
logger.error(`Failed to delete server image ${serverImagePath}:`, error);
}
}
if (generationTasks.length > 0) {
logger.info(`--- Starting parallel generation of ${generationTasks.length} tasks across ${servers.length} servers ---`);
const workers = servers.map((server, index) => worker(index + 1, server, generationTasks));
await Promise.all(workers);
logger.info("--- Finished parallel generation ---");
}
}
logger.info("--- Finished video generation ---");
}
})();