ken/RandomVideoMaker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

save changes

Browse Source
This commit is contained in:
Ken Yasue
2025-08-27 12:07:51 +02:00
parent d4028e8f7f
commit 153e50a356
3 changed files with 462 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
src/comfyworkflows/generate_image_mix_style.json Normal file
View File

@ -0,0 +1,240 @@
{
"1": {
"inputs": {
"clip_name1": "clip_l.safetensors",
"clip_name2": "t5xxl_fp16.safetensors",
"type": "flux",
"device": "default"
},
"class_type": "DualCLIPLoader",
"_meta": {
"title": "DualCLIPLoader"
}
},
"2": {
"inputs": {
"unet_name": "flux1-krea-dev_fp8_scaled.safetensors",
"weight_dtype": "default"
},
"class_type": "UNETLoader",
"_meta": {
"title": "Load Diffusion Model"
}
},
"3": {
"inputs": {
"width": 320,
"height": 640,
"batch_size": 1
},
"class_type": "EmptySD3LatentImage",
"_meta": {
"title": "EmptySD3LatentImage"
}
},
"4": {
"inputs": {
"seed": 803508963683741,
"steps": 20,
"cfg": 1,
"sampler_name": "euler",
"scheduler": "simple",
"denoise": 1,
"model": [
"2",
0
],
"positive": [
"18",
0
],
"negative": [
"5",
0
],
"latent_image": [
"3",
0
]
},
"class_type": "KSampler",
"_meta": {
"title": "KSampler"
}
},
"5": {
"inputs": {
"conditioning": [
"14",
0
]
},
"class_type": "ConditioningZeroOut",
"_meta": {
"title": "ConditioningZeroOut"
}
},
"6": {
"inputs": {
"samples": [
"4",
0
],
"vae": [
"12",
0
]
},
"class_type": "VAEDecode",
"_meta": {
"title": "VAE Decode"
}
},
"8": {
"inputs": {
"style_model_name": "flux1-redux-dev.safetensors"
},
"class_type": "StyleModelLoader",
"_meta": {
"title": "Load Style Model"
}
},
"9": {
"inputs": {
"crop": "center",
"clip_vision": [
"10",
0
],
"image": [
"13",
0
]
},
"class_type": "CLIPVisionEncode",
"_meta": {
"title": "CLIP Vision Encode"
}
},
"10": {
"inputs": {
"clip_name": "sigclip_vision_patch14_384.safetensors"
},
"class_type": "CLIPVisionLoader",
"_meta": {
"title": "Load CLIP Vision"
}
},
"11": {
"inputs": {
"strength": 0.20000000000000004,
"conditioning": [
"14",
0
],
"style_model": [
"8",
0
],
"clip_vision_output": [
"9",
0
]
},
"class_type": "ApplyStyleModelAdjust",
"_meta": {
"title": "Apply Style Model (Adjusted)"
}
},
"12": {
"inputs": {
"vae_name": "ae.safetensors"
},
"class_type": "VAELoader",
"_meta": {
"title": "Load VAE"
}
},
"13": {
"inputs": {
"image": "fd2dc3bbc879703b03abf892d4189667.jpg"
},
"class_type": "LoadImage",
"_meta": {
"title": "Load Image 1"
}
},
"14": {
"inputs": {
"text": "Ethereal realistic girl with flowing blue hair, glowing sparkles and stardust, elegant backless dress shimmering with cosmic light, dreamy profile pose, soft glowing skin, fantasy night atmosphere, luminous and magical aesthetic",
"clip": [
"1",
0
]
},
"class_type": "CLIPTextEncode",
"_meta": {
"title": "CLIP Text Encode (Prompt)"
}
},
"15": {
"inputs": {
"filename_prefix": "STYLEDVIDEOMAKER",
"images": [
"6",
0
]
},
"class_type": "SaveImage",
"_meta": {
"title": "Save Image"
}
},
"16": {
"inputs": {
"image": "a09ee3d44fff05f20c88976555f8fa10.jpg"
},
"class_type": "LoadImage",
"_meta": {
"title": "Load Image 2"
}
},
"17": {
"inputs": {
"crop": "center",
"clip_vision": [
"10",
0
],
"image": [
"16",
0
]
},
"class_type": "CLIPVisionEncode",
"_meta": {
"title": "CLIP Vision Encode"
}
},
"18": {
"inputs": {
"strength": 0.20000000000000004,
"conditioning": [
"11",
0
],
"style_model": [
"8",
0
],
"clip_vision_output": [
"17",
0
]
},
"class_type": "ApplyStyleModelAdjust",
"_meta": {
"title": "Apply Style Model (Adjusted)"
}
}
}

78
src/lib/image-generator-mix-style.ts Normal file
View File

@ -0,0 +1,78 @@
import * as fs from 'fs/promises';
import * as path from 'path';
import axios from 'axios';
import dotenv from 'dotenv';
dotenv.config();
interface ImageSize {
width: number;
height: number;
}
async function generateImage(
prompt: string,
imageName1: string,
imageName2: string,
newFileName: string,
comfyBaseUrl: string,
comfyOutputDir: string,
size: ImageSize = { width: 720, height: 1280 }
): Promise<string> {
const COMFY_BASE_URL = comfyBaseUrl.replace(/\/$/, '');
const COMFY_OUTPUT_DIR = comfyOutputDir;
let workflow;
workflow = JSON.parse(await fs.readFile('src/comfyworkflows/generate_image_mix_style.json', 'utf-8'));
workflow['14']['inputs']['text'] = prompt;
// Set image name
workflow['13']['inputs']['image'] = imageName1;
// Set image name
workflow['16']['inputs']['image'] = imageName2;
workflow['3']['inputs']['width'] = size.width;
workflow['3']['inputs']['height'] = size.height;
const response = await axios.post(`${COMFY_BASE_URL}/prompt`, { prompt: workflow });
const promptId = response.data.prompt_id;
let history;
do {
await new Promise(resolve => setTimeout(resolve, 1000));
const historyResponse = await axios.get(`${COMFY_BASE_URL}/history/${promptId}`);
history = historyResponse.data[promptId];
} while (!history || Object.keys(history.outputs).length === 0);
const files = await fs.readdir(COMFY_OUTPUT_DIR!);
const generatedFiles = files.filter(file => file.startsWith('STYLEDVIDEOMAKER'));
const fileStats = await Promise.all(
generatedFiles.map(async (file) => {
const stat = await fs.stat(path.join(COMFY_OUTPUT_DIR!, file));
return { file, mtime: stat.mtime };
})
);
fileStats.sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
const latestFile = fileStats[0].file;
const newFilePath = path.resolve('./generated', newFileName);
await fs.mkdir('./generated', { recursive: true });
const sourcePath = path.join(COMFY_OUTPUT_DIR!, latestFile);
try {
await fs.unlink(newFilePath);
} catch (err) {
// ignore if not exists
}
await fs.copyFile(sourcePath, newFilePath);
return newFilePath;
}
export { generateImage };

240
src/piterest_styletransfer_video.ts
View File

@ -1,6 +1,7 @@
import { downloadImagesFromPinterestPin } from './lib/downloader';
import { callOpenAIWithFile } from './lib/openai';
import { generateStyledVideo } from './lib/video-generator-styled';
import { generateVideo } from './lib/video-generator';
import { generateImage } from './lib/image-generator-mix-style';
import { logger } from './lib/logger';
import * as fs from 'fs/promises';
import dotenv from 'dotenv';
@ -28,7 +29,8 @@ interface GenerationTask {
imagePrompt: string;
videoPrompt: string;
imageFileName: string;
renamedImagePath: string;
renamedImagePaths: string[];
generatedImagePath?: string;
genre: string;
subGenre: string;
scene: string;
@ -36,17 +38,23 @@ interface GenerationTask {
camera: string;
}
async function getPromptsForImage(imagePath: string, pinUrl: string, genre: string, subGenre: string): Promise<GenerationTask | null> {
async function getPromptsForImage(imagePaths: string[], pinUrl: string, genre: string, subGenre: string): Promise<GenerationTask | null> {
const pinId = pinUrl.split('/').filter(Boolean).pop() || `pin_${Date.now()}`;
const timestamp = new Date().getTime();
const imageFileName = `${pinId}_${timestamp}.png`;
const renamedImagePath = path.join(path.dirname(imagePath), imageFileName);
const renamedImagePaths = [];
for (let i = 0; i < imagePaths.length; i++) {
const renamedPath = path.join(path.dirname(imagePaths[i]), `${pinId}_${timestamp}_${i}.png`);
await fs.rename(imagePaths[i], renamedPath);
renamedImagePaths.push(renamedPath);
}
logger.debug(`Renamed source images to: ${renamedImagePaths.join(', ')}`);
const imageForPrompt = renamedImagePaths[Math.floor(Math.random() * renamedImagePaths.length)];
try {
await fs.rename(imagePath, renamedImagePath);
logger.debug(`Renamed ${imagePath} to ${renamedImagePath}`);
const promptResponse = await callOpenAIWithFile(renamedImagePath,
const promptResponse = await callOpenAIWithFile(imageForPrompt,
`Analyze the provided image and generate the following:
1. 'scene': A description of the image's environment.
2. 'action': A description of the main action occurring in the image.
@ -66,100 +74,70 @@ async function getPromptsForImage(imagePath: string, pinUrl: string, genre: stri
---
`);
const { scene, action, camera, image_prompt: imagePrompt, video_prompt: videoPrompt } = promptResponse;
logger.info(`Image prompt for ${renamedImagePath}:`, imagePrompt);
logger.info(`Video prompt for ${renamedImagePath}:`, videoPrompt);
logger.info(`Image prompt for ${imageForPrompt}:`, imagePrompt);
logger.info(`Video prompt for ${imageForPrompt}:`, videoPrompt);
return { pinUrl, imagePrompt, videoPrompt, imageFileName, renamedImagePath, genre, subGenre, scene, action, camera };
return { pinUrl, imagePrompt, videoPrompt, imageFileName, renamedImagePaths, genre, subGenre, scene, action, camera };
} catch (error) {
logger.error(`Failed to get prompts for ${renamedImagePath}:`, error);
try {
await fs.unlink(renamedImagePath);
} catch (cleanupError) {
// ignore
logger.error(`Failed to get prompts for ${imageForPrompt}:`, error);
for (const p of renamedImagePaths) {
try {
await fs.unlink(p);
} catch (cleanupError) {
// ignore
}
}
return null;
}
}
async function generateImageAndVideo(task: GenerationTask, server: { baseUrl: string; outputDir: string; }): Promise<{ imagePath: string; videoPath: string; } | null> {
const { imagePrompt, videoPrompt, imageFileName, renamedImagePath } = task;
async function generateImageForTask(task: GenerationTask, server: { baseUrl: string; outputDir: string; }): Promise<string | null> {
const { imagePrompt, imageFileName, renamedImagePaths } = task;
const { baseUrl, outputDir } = server;
const inputDir = outputDir.replace("output", "input");
const sourceFileNames: string[] = [];
try {
const destPath = path.join(inputDir, imageFileName);
await fs.copyFile(renamedImagePath, destPath);
logger.info(`Copied ${renamedImagePath} to ${destPath}`);
for (const sourcePath of renamedImagePaths) {
const fileName = path.basename(sourcePath);
const destPath = path.join(inputDir, fileName);
await fs.copyFile(sourcePath, destPath);
sourceFileNames.push(fileName);
logger.info(`Copied ${sourcePath} to ${destPath}`);
}
const videoFileName = imageFileName.replace('.png', '.mp4');
const { videoPath, imagePath } = await generateStyledVideo(
const generatedImagePath = await generateImage(
imagePrompt,
videoPrompt,
sourceFileNames[0],
sourceFileNames[1],
imageFileName,
videoFileName,
baseUrl,
outputDir,
{ width: 720, height: 1280 }
);
path.join(outputDir, videoFileName);
return { imagePath: imagePath, videoPath };
return generatedImagePath;
} catch (error) {
logger.error(`Failed to generate styled video for ${imageFileName} on server ${baseUrl}:`, error);
logger.error(`Failed to generate image for ${imageFileName} on server ${baseUrl}:`, error);
return null;
} finally {
try {
await fs.unlink(renamedImagePath);
logger.debug(`Deleted renamed source image: ${renamedImagePath}`);
} catch (error) {
logger.error(`Failed to delete renamed source image ${renamedImagePath}:`, error);
for (const sourcePath of renamedImagePaths) {
try {
await fs.unlink(sourcePath);
logger.debug(`Deleted source image: ${sourcePath}`);
} catch (error) {
logger.error(`Failed to delete source image ${sourcePath}:`, error);
}
}
}
}
async function worker(id: number, server: { baseUrl: string; outputDir: string; }, taskQueue: GenerationTask[]) {
logger.info(`Worker ${id} started for server ${server.baseUrl}`);
while (taskQueue.length > 0) {
const task = taskQueue.shift();
if (task) {
logger.info(`Worker ${id} processing task: ${task.imageFileName}`);
const result = await generateImageAndVideo(task, server);
if (result) {
try {
const videoData = {
genre: task.genre,
sub_genre: task.subGenre,
scene: task.scene,
action: task.action,
camera: task.camera,
image_prompt: task.imagePrompt,
video_prompt: task.videoPrompt,
image_path: result.imagePath,
video_path: result.videoPath,
};
const videoId = await VideoModel.create(videoData);
logger.info(`Successfully saved video record to database with ID: ${videoId}`);
const newImageName = `${videoId}_${task.genre}_${task.subGenre}${path.extname(result.imagePath)}`;
const newVideoName = `${videoId}_${task.genre}_${task.subGenre}${path.extname(result.videoPath)}`;
const newImagePath = path.join(path.dirname(result.imagePath), newImageName);
const newVideoPath = path.join(path.dirname(result.videoPath), newVideoName);
await fs.rename(result.imagePath, newImagePath);
await fs.rename(result.videoPath, newVideoPath);
await VideoModel.update(videoId, {
image_path: newImagePath,
video_path: newVideoPath,
});
logger.info(`Renamed files and updated database record for video ID: ${videoId}`);
} catch (error) {
logger.error('Failed to save video record to database or rename files:', error);
}
for (const fileName of sourceFileNames) {
try {
const serverPath = path.join(inputDir, fileName);
await fs.unlink(serverPath);
logger.debug(`Deleted server image: ${serverPath}`);
} catch (error) {
logger.error(`Failed to delete server image ${fileName}:`, error);
}
}
}
logger.info(`Worker ${id} finished.`);
}
async function getPinUrlFromPinterest(keyword: string): Promise<string | null> {
@ -212,6 +190,7 @@ async function getPinUrlFromPinterest(keyword: string): Promise<string | null> {
}
while (true) {
const generationTasks: GenerationTask[] = [];
for (const genreSubGenre of keywords) {
@ -235,8 +214,6 @@ async function getPinUrlFromPinterest(keyword: string): Promise<string | null> {
continue;
}
const generationTasks: GenerationTask[] = [];
logger.info(`--- Starting processing for pin: ${pin} ---`);
const downloadedImagePaths = await downloadImagesFromPinterestPin(pin);
if (downloadedImagePaths.length === 0) {
@ -244,36 +221,107 @@ async function getPinUrlFromPinterest(keyword: string): Promise<string | null> {
continue;
}
const selectedImages = downloadedImagePaths.sort(() => 0.5 - Math.random()).slice(0, 1);
const selectedImages = downloadedImagePaths.sort(() => 0.5 - Math.random()).slice(0, 2);
logger.info(`--- Randomly selected ${selectedImages.length} images for processing ---`);
for (const imagePath of selectedImages) {
const pinId = path.basename(imagePath, path.extname(imagePath)).split('_related_')[0];
if (selectedImages.length === 2) {
const pinId = path.basename(selectedImages[0], path.extname(selectedImages[0])).split('_related_')[0];
const pinUrl = `https://www.pinterest.com/pin/${pinId}/`;
const task = await getPromptsForImage(imagePath, pinUrl, genre, subGenre);
const task = await getPromptsForImage(selectedImages, pinUrl, genre, subGenre);
if (task) {
generationTasks.push(task);
}
}
const unselectedImages = downloadedImagePaths.filter(p => !selectedImages.includes(p));
for (const imagePath of unselectedImages) {
try {
} else {
logger.warn(`Skipping pin ${pin} as it did not have at least 2 images.`);
for (const imagePath of selectedImages) {
await fs.unlink(imagePath);
logger.debug(`Deleted unselected image: ${imagePath}`);
} catch (error) {
logger.error(`Failed to delete unselected image ${imagePath}:`, error);
}
}
}
if (generationTasks.length > 0) {
logger.info(`--- Starting parallel generation of ${generationTasks.length} tasks across ${servers.length} servers ---`);
const workers = servers.map((server, index) => worker(index + 1, server, generationTasks));
await Promise.all(workers);
logger.info("--- Finished parallel generation ---");
// --- Image Generation Phase ---
logger.info(`--- Starting image generation for ${generationTasks.length} tasks ---`);
for (const task of generationTasks) {
const server = servers[Math.floor(Math.random() * servers.length)];
const imagePath = await generateImageForTask(task, server);
if (imagePath) {
task.generatedImagePath = imagePath;
}
}
logger.info("--- Finished image generation ---");
// --- Video Generation Phase ---
logger.info(`--- Starting video generation for ${generationTasks.length} tasks ---`);
for (const task of generationTasks) {
if (!task.generatedImagePath) {
logger.warn(`Skipping video generation for task ${task.imageFileName} as it has no generated image.`);
continue;
}
const server = servers[Math.floor(Math.random() * servers.length)];
const inputDir = server.outputDir.replace("output", "input");
const generatedImageName = path.basename(task.generatedImagePath);
const serverImagePath = path.join(inputDir, generatedImageName);
try {
await fs.copyFile(task.generatedImagePath, serverImagePath);
logger.info(`Copied ${task.generatedImagePath} to ${serverImagePath}`);
const videoFileName = task.imageFileName.replace('.png', '.mp4');
const videoPath = await generateVideo(
task.videoPrompt,
generatedImageName,
videoFileName,
server.baseUrl,
server.outputDir,
{ width: 720, height: 1280 }
);
if (videoPath) {
const videoData = {
genre: task.genre,
sub_genre: task.subGenre,
scene: task.scene,
action: task.action,
camera: task.camera,
image_prompt: task.imagePrompt,
video_prompt: task.videoPrompt,
image_path: task.generatedImagePath,
video_path: videoPath,
};
const videoId = await VideoModel.create(videoData);
logger.info(`Successfully saved video record to database with ID: ${videoId}`);
const newImageName = `${videoId}_${task.genre}_${task.subGenre}${path.extname(task.generatedImagePath)}`;
const newVideoName = `${videoId}_${task.genre}_${task.subGenre}${path.extname(videoPath)}`;
const newImagePath = path.join(path.dirname(task.generatedImagePath), newImageName);
const newVideoPath = path.join(path.dirname(videoPath), newVideoName);
await fs.rename(task.generatedImagePath, newImagePath);
await fs.rename(videoPath, newVideoPath);
await VideoModel.update(videoId, {
image_path: newImagePath,
video_path: newVideoPath,
});
logger.info(`Renamed files and updated database record for video ID: ${videoId}`);
await fs.unlink(newImagePath);
logger.info(`Deleted paired image: ${newImagePath}`);
}
} catch (error) {
logger.error('An error occurred during video generation or database operations:', error);
} finally {
try {
await fs.unlink(serverImagePath);
logger.debug(`Deleted server image: ${serverImagePath}`);
} catch (error) {
logger.error(`Failed to delete server image ${serverImagePath}:`, error);
}
}
}
logger.info("--- Finished video generation ---");
}
})();