ken/RandomVideoMaker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://git.yasue.org/ken/RandomVideoMaker

Browse Source
This commit is contained in:
Ken Yasue
2025-09-25 21:49:49 +02:00
parent 28aa838937 4ccfd9ef3a
commit d362f9e0ca
8 changed files with 676 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
.clinerules/generators.md Normal file
View File

@ -0,0 +1,18 @@
#Image and Video generation
## Simple image generation with prompt
Use this file src\lib\image-generator.ts
## Image generation with existing image
Use this file src\lib\image-generator-face.ts
## Image converting
Use this file src\lib\image-converter.ts
## Vide generation with static image
Use this ilfe src\lib\video-generator.ts
Everything when generator need to use existing image you have to copy to input folder to the server and use only filename.
For example if
inputfolderFullpath = SERVER1_COMFY_OUTPUT_DIR.replece("output","input");
Then copy image to the path, and pass only filename to generator function

17
.clinerules/llm.md Normal file
View File

@ -0,0 +1,17 @@
# LLM API
## LMstudio
Use this file src\lib\lmstudio.ts
- async function callLmstudio(prompt: string): Promise<any> {
for just run prompt
- async function callLMStudioAPIWithFile(imagePath: string, prompt: string): Promise<any> {
for send file to llm
## OpenAI
Use this file src\lib\openai.ts
- async function callOpenAI(prompt: string): Promise<any>
for just run prompt
- async function callOpenAIWithFile(imagePath: string, prompt: string): Promise<any>
for send file to llm

207
src/comfyworkflows/cloth_extractor.json Normal file
View File

@ -0,0 +1,207 @@
{
"1": {
"inputs": {
"unet_name": "qwen_image_edit_2509_fp8_e4m3fn.safetensors",
"weight_dtype": "default"
},
"class_type": "UNETLoader",
"_meta": {
"title": "Load Diffusion Model"
}
},
"2": {
"inputs": {
"clip_name": "qwen_2.5_vl_7b_fp8_scaled.safetensors",
"type": "qwen_image",
"device": "default"
},
"class_type": "CLIPLoader",
"_meta": {
"title": "Load CLIP"
}
},
"3": {
"inputs": {
"vae_name": "qwen_image_vae.safetensors"
},
"class_type": "VAELoader",
"_meta": {
"title": "Load VAE"
}
},
"4": {
"inputs": {
"lora_name": "Qwen-Image-Lightning-8steps-V2.0.safetensors",
"strength_model": 1,
"model": [
"1",
0
]
},
"class_type": "LoraLoaderModelOnly",
"_meta": {
"title": "LoraLoaderModelOnly"
}
},
"5": {
"inputs": {
"conditioning": [
"11",
0
]
},
"class_type": "ConditioningZeroOut",
"_meta": {
"title": "ConditioningZeroOut"
}
},
"7": {
"inputs": {
"seed": 1088883674457465,
"steps": 8,
"cfg": 1,
"sampler_name": "euler",
"scheduler": "beta",
"denoise": 1,
"model": [
"66",
0
],
"positive": [
"11",
0
],
"negative": [
"5",
0
],
"latent_image": [
"11",
6
]
},
"class_type": "KSampler",
"_meta": {
"title": "KSampler"
}
},
"8": {
"inputs": {
"samples": [
"7",
0
],
"vae": [
"3",
0
]
},
"class_type": "VAEDecode",
"_meta": {
"title": "VAE Decode"
}
},
"11": {
"inputs": {
"prompt": [
"21",
0
],
"enable_resize": true,
"enable_vl_resize": true,
"upscale_method": "lanczos",
"crop": "disabled",
"instruction": "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
"clip": [
"2",
0
],
"vae": [
"3",
0
],
"image1": [
"24",
0
]
},
"class_type": "TextEncodeQwenImageEditPlus_lrzjason",
"_meta": {
"title": "TextEncodeQwenImageEditPlus 小志Jason(xiaozhijason)"
}
},
"20": {
"inputs": {
"filename_prefix": "qwenedit",
"images": [
"8",
0
]
},
"class_type": "SaveImage",
"_meta": {
"title": "Save Image"
}
},
"21": {
"inputs": {
"value": "extract the outfit onto a white background"
},
"class_type": "PrimitiveStringMultiline",
"_meta": {
"title": "String (Multiline)"
}
},
"24": {
"inputs": {
"measurement": "pixels",
"width": 720,
"height": 1280,
"fit": "contain",
"method": "nearest-exact",
"image": [
"64",
0
]
},
"class_type": "Image Resize (rgthree)",
"_meta": {
"title": "Image Resize (rgthree)"
}
},
"64": {
"inputs": {
"image": "3096293489212792_1758825204441_2.png"
},
"class_type": "LoadImage",
"_meta": {
"title": "Load Image"
}
},
"65": {
"inputs": {
"images": [
"24",
0
]
},
"class_type": "PreviewImage",
"_meta": {
"title": "Preview Image"
}
},
"66": {
"inputs": {
"lora_name": "extract-outfit_v3.safetensors",
"strength_model": 1,
"model": [
"4",
0
]
},
"class_type": "LoraLoaderModelOnly",
"_meta": {
"title": "LoraLoaderModelOnly"
}
}
}

24
src/comfyworkflows/edit_image_2_qwen.json
View File

@ -57,7 +57,7 @@
},
"7": {
"inputs": {
"seed": 135126386717887,
"seed": 838097333311955,
"steps": 8,
"cfg": 1,
"sampler_name": "euler",
@ -158,7 +158,7 @@
},
"21": {
"inputs": {
"value": "change the face and hairstyle of image1 to image2"
"value": "change camera angle to closeup face from image1, change background to light gray with faing gradient, change face angle to look at directry look at camera"
},
"class_type": "PrimitiveStringMultiline",
"_meta": {
@ -170,10 +170,10 @@
"measurement": "pixels",
"width": 720,
"height": 1280,
"fit": "pad",
"fit": "contain",
"method": "nearest-exact",
"image": [
"63",
"64",
0
]
},
@ -182,13 +182,25 @@
"title": "Image Resize (rgthree)"
}
},
"63": {
"64": {
"inputs": {
"image": "00d3c030856044bb99b804b4ef88f4be.jpg"
"image": "1337074888177434_1758776251440_2.png"
},
"class_type": "LoadImage",
"_meta": {
"title": "Load Image"
}
},
"65": {
"inputs": {
"images": [
"24",
0
]
},
"class_type": "PreviewImage",
"_meta": {
"title": "Preview Image"
}
}
}

125
src/lib/image-converter.ts
View File

@ -12,7 +12,7 @@ interface ImageSize {
async function convertImage(
prompt: string,
newFileName: string,
baseFileName: string,
comfyBaseUrl: string,
comfyOutputDir: string,
size: ImageSize = { width: 720, height: 1280 }
@ -25,7 +25,7 @@ async function convertImage(
workflow['21']['inputs']['value'] = prompt;
workflow['24']['inputs']['width'] = size.width;
workflow['24']['inputs']['height'] = size.height;
workflow['64']['inputs']['image'] = newFileName;
workflow['64']['inputs']['image'] = baseFileName;
const response = await axios.post(`${COMFY_BASE_URL}/prompt`, { prompt: workflow });
const promptId = response.data.prompt_id;
@ -50,7 +50,7 @@ async function convertImage(
fileStats.sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
const latestFile = fileStats[0].file;
const newFilePath = path.resolve('./generated', newFileName);
const newFilePath = path.resolve('./generated', baseFileName);
await fs.mkdir('./generated', { recursive: true });
@ -66,4 +66,121 @@ async function convertImage(
return newFilePath;
}
export { convertImage };
async function convertImageWithFile(
prompt: string,
baseFileName: string,
secondFileName: string,
comfyBaseUrl: string,
comfyOutputDir: string,
size: ImageSize = { width: 720, height: 1280 }
): Promise<string> {
const COMFY_BASE_URL = comfyBaseUrl.replace(/\/$/, '');
const COMFY_OUTPUT_DIR = comfyOutputDir;
let workflow;
workflow = JSON.parse(await fs.readFile('src/comfyworkflows/edit_image_2_qwen.json', 'utf-8'));
workflow['21']['inputs']['value'] = prompt;
workflow['24']['inputs']['width'] = size.width;
workflow['24']['inputs']['height'] = size.height;
workflow['64']['inputs']['image'] = baseFileName;
workflow['15']['inputs']['image'] = secondFileName;
const response = await axios.post(`${COMFY_BASE_URL}/prompt`, { prompt: workflow });
const promptId = response.data.prompt_id;
let history;
do {
await new Promise(resolve => setTimeout(resolve, 1000));
const historyResponse = await axios.get(`${COMFY_BASE_URL}/history/${promptId}`);
history = historyResponse.data[promptId];
} while (!history || Object.keys(history.outputs).length === 0);
const files = await fs.readdir(COMFY_OUTPUT_DIR!);
const generatedFiles = files.filter(file => file.startsWith('qwenedit'));
const fileStats = await Promise.all(
generatedFiles.map(async (file) => {
const stat = await fs.stat(path.join(COMFY_OUTPUT_DIR!, file));
return { file, mtime: stat.mtime };
})
);
fileStats.sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
const latestFile = fileStats[0].file;
const newFilePath = path.resolve('./generated', baseFileName);
await fs.mkdir('./generated', { recursive: true });
const sourcePath = path.join(COMFY_OUTPUT_DIR!, latestFile);
try {
await fs.unlink(newFilePath);
} catch (err) {
// ignore if not exists
}
await fs.copyFile(sourcePath, newFilePath);
return newFilePath;
}
export async function extractCloth(
prompt: string,
baseFileName: string,
comfyBaseUrl: string,
comfyOutputDir: string,
size: ImageSize = { width: 720, height: 1280 }
): Promise<string> {
const COMFY_BASE_URL = comfyBaseUrl.replace(/\/$/, '');
const COMFY_OUTPUT_DIR = comfyOutputDir;
let workflow;
workflow = JSON.parse(await fs.readFile('src/comfyworkflows/cloth_extractor.json', 'utf-8'));
workflow['21']['inputs']['value'] = prompt;
workflow['24']['inputs']['width'] = size.width;
workflow['24']['inputs']['height'] = size.height;
workflow['64']['inputs']['image'] = baseFileName;
const response = await axios.post(`${COMFY_BASE_URL}/prompt`, { prompt: workflow });
const promptId = response.data.prompt_id;
let history;
do {
await new Promise(resolve => setTimeout(resolve, 1000));
const historyResponse = await axios.get(`${COMFY_BASE_URL}/history/${promptId}`);
history = historyResponse.data[promptId];
} while (!history || Object.keys(history.outputs).length === 0);
const files = await fs.readdir(COMFY_OUTPUT_DIR!);
const generatedFiles = files.filter(file => file.startsWith('qwenedit'));
const fileStats = await Promise.all(
generatedFiles.map(async (file) => {
const stat = await fs.stat(path.join(COMFY_OUTPUT_DIR!, file));
return { file, mtime: stat.mtime };
})
);
fileStats.sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
const latestFile = fileStats[0].file;
const newFilePath = path.resolve('./generated', baseFileName);
await fs.mkdir('./generated', { recursive: true });
const sourcePath = path.join(COMFY_OUTPUT_DIR!, latestFile);
try {
await fs.unlink(newFilePath);
} catch (err) {
// ignore if not exists
}
await fs.copyFile(sourcePath, newFilePath);
return newFilePath;
}
export { convertImage, convertImageWithFile };

122
src/tools/face_normalizer.ts Normal file
View File

@ -0,0 +1,122 @@
import * as fs from 'fs';
import { promises as fsPromises } from 'fs';
import * as path from 'path';
import * as crypto from 'crypto';
import dotenv from 'dotenv';
import { callLMStudioAPIWithFile } from '../lib/lmstudio';
import { generateImage } from '../lib/image-generator';
import { convertImageWithFile } from '../lib/image-converter';
dotenv.config();
const inputDir = path.resolve('./input');
const outputDir = path.resolve('./generated');
async function main() {
const comfyBaseUrl = process.env.SERVER1_COMFY_BASE_URL;
const comfyOutputDir = process.env.SERVER1_COMFY_OUTPUT_DIR;
if (!comfyBaseUrl || !comfyOutputDir) {
throw new Error('SERVER1_COMFY_BASE_URL and SERVER1_COMFY_OUTPUT_DIR must be set in .env file');
}
const comfyInputDir = comfyOutputDir.replace("output", "input");
// Ensure output directory exists
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
const files = fs.readdirSync(inputDir);
for (const file of files) {
const imagePath = path.join(inputDir, file);
// Calculate file hash
const fileBuffer = fs.readFileSync(imagePath);
const hashSum = crypto.createHash('sha256');
hashSum.update(fileBuffer);
const hexHash = hashSum.digest('hex');
const genFileName = `${hexHash}_gen.png`;
const convertedFileName = `${hexHash}_converted.png`;
const genFilePath = path.join(outputDir, genFileName);
const convertedFilePath = path.join(outputDir, convertedFileName);
if (fs.existsSync(genFilePath) && fs.existsSync(convertedFilePath)) {
console.log(`Skipping ${file} as it has already been processed.`);
continue;
}
console.log(`Processing ${imagePath}...`);
// 1. Ask LLM to make a prompt for this photo
const llmPrompt = `
Extract the following information from the attached image: face detail, hairstyle, eye color, hair color.
Output instruction:
{ result: "face detail, hairstyle, eye color, hair color" }
`;
const extractedInfoJSON = await callLMStudioAPIWithFile(imagePath, llmPrompt);
if (!extractedInfoJSON) {
console.error(`Could not get prompt from LLM for ${imagePath}`);
continue;
}
const extractedInfo = extractedInfoJSON.result;
console.log(`LLM response: ${extractedInfo}`);
// 2. Generate photo using the prompt
const prefix = "Highly detailed face portrait of 20 years old girl, light gray background with faint gradient, look at camera in front";
const finalPrompt = prefix + extractedInfo;
console.log(`Generating image with prompt: ${finalPrompt}`);
const generatedImagePath = await generateImage(
finalPrompt,
genFileName,
comfyBaseUrl,
comfyOutputDir,
'flux'
);
if (generatedImagePath) {
console.log(`Generated image saved to: ${generatedImagePath}`);
// 3. Convert the image
const generatedImageFileName = path.basename(generatedImagePath);
const originalImageFileName = file;
const destGeneratedImagePath = path.join(comfyInputDir, generatedImageFileName);
const destOriginalImagePath = path.join(comfyInputDir, originalImageFileName);
// Copy files to comfy input directory
await fsPromises.copyFile(generatedImagePath, destGeneratedImagePath);
await fsPromises.copyFile(imagePath, destOriginalImagePath);
console.log(`Copied ${generatedImageFileName} and ${originalImageFileName} to ${comfyInputDir}`);
const conversionPrompt = "Replace the hairs of image1 with hairs of image2, change face of image1 with face of image2.";
const convertedResultPath = await convertImageWithFile(
conversionPrompt,
generatedImageFileName,
originalImageFileName,
comfyBaseUrl,
comfyOutputDir
);
if (convertedResultPath) {
// Rename the output file to the final converted name
await fsPromises.rename(convertedResultPath, convertedFilePath);
console.log(`Converted image saved to: ${convertedFilePath}`);
} else {
console.error(`Failed to convert image for ${originalImageFileName}`);
}
} else {
console.error(`Failed to generate image for prompt: ${finalPrompt}`);
}
}
}
main().catch(console.error);

173
src/tools/pinterest_cloth_extraction.ts Normal file
View File

@ -0,0 +1,173 @@
import dotenv from 'dotenv';
import * as fs from 'fs/promises';
import * as path from 'path';
import { logger } from '../lib/logger';
import { getPinUrlFromPinterest, downloadImageFromPin } from '../lib/pinterest';
import { convertImage, extractCloth } from '../lib/image-converter';
dotenv.config();
const KEYWORDS = [
'teen skirt outfit',
'teen casual cute outfit',
'cute outfit',
'summer dress',
'elegant dress',
'bohemian dress',
'ball gown dress',
'cosplay outfit',
'vintage skirt outfit',
'casual skirt outfit'];
const TARGET_COUNT = Number(process.env.IMAGE_COUNT || 20);
const PROMPT =
`Change pose of the person in the image1 to stantind in front of camera, with a smile, full body visible, wearing a fashionable outfit suitable for a casual day out. The background should be a white. Ensure the lighting is bright and natural, highlighting the details of the outfit and the person's features.
`;
const PROMPT2 =
`Extract clothes from image1, put it on the white backgroud
`;
type ServerCfg = { baseUrl: string; outputDir: string; inputDir: string };
function getServerConfig(): ServerCfg {
const candidates = [
{ baseUrl: process.env.SERVER1_COMFY_BASE_URL, outputDir: process.env.SERVER1_COMFY_OUTPUT_DIR },
//{ baseUrl: process.env.SERVER2_COMFY_BASE_URL, outputDir: process.env.SERVER2_COMFY_OUTPUT_DIR },
].filter((s): s is { baseUrl: string; outputDir: string } => !!s.baseUrl && !!s.outputDir);
if (candidates.length === 0) {
throw new Error(
'No ComfyUI server configured. Please set SERVER1_COMFY_BASE_URL/OUTPUT_DIR or SERVER2_COMFY_BASE_URL/OUTPUT_DIR in .env'
);
}
const chosen = candidates[0];
const inputDir = chosen.outputDir.replace('output', 'input');
return { ...chosen, inputDir };
}
function sleep(ms: number) {
return new Promise((res) => setTimeout(res, ms));
}
async function ensureDir(p: string) {
try {
await fs.mkdir(p, { recursive: true });
} catch {
// ignore
}
}
async function collectImages(keyword: string, total: number): Promise<string[]> {
const results: string[] = [];
// ensure local download dir exists (pinterest.ts also ensures it, but harmless here)
await ensureDir(path.join(process.cwd(), 'download'));
while (results.length < total) {
try {
const pinUrl = await getPinUrlFromPinterest(keyword);
if (!pinUrl) {
logger.warn('No pin URL found, retrying...');
await sleep(1500);
continue;
}
const remaining = total - results.length;
// attempt to grab up to 5 per pin to reduce churn
const batchTarget = Math.min(5, remaining);
const imgs = await downloadImageFromPin(pinUrl, batchTarget);
if (imgs && imgs.length > 0) {
results.push(...imgs);
logger.info(`Downloaded ${imgs.length} image(s) from ${pinUrl}.Progress: ${results.length}/${total}`);
} else {
logger.warn(`Pin yielded no downloadable images: ${pinUrl}`);
}
await sleep(1000 + Math.random() * 1000);
} catch (err) {
logger.error('Error while collecting images:', err);
await sleep(2000);
}
}
return results.slice(0, total);
}
async function processImages(imagePaths: string[], server: ServerCfg) {
await ensureDir(server.inputDir);
for (const localImagePath of imagePaths) {
const baseName = path.basename(localImagePath);
const serverInputPath = path.join(server.inputDir, baseName);
try {
// copy source image into ComfyUI input dir so workflow node can find it by filename
await fs.copyFile(localImagePath, serverInputPath);
logger.info(`Copied ${localImagePath} -> ${serverInputPath}`);
// Run conversion (sequential to avoid output race conditions)
const generatedPath1 = await convertImage(
PROMPT,
baseName,
server.baseUrl,
server.outputDir,
{ width: 720, height: 1280 } // portrait
);
await fs.copyFile(generatedPath1, serverInputPath);
const baseName2 = path.basename(localImagePath);
const generatedPath2 = await extractCloth(
PROMPT2,
baseName2,
server.baseUrl,
server.outputDir,
{ width: 720, height: 1280 } // portrait
);
logger.info(`Generated image: ${generatedPath2}`);
} catch (err) {
logger.error(`Failed to convert ${localImagePath}:`, err);
} finally {
// cleanup both server input copy and local downloaded file
try {
await fs.unlink(serverInputPath);
} catch { }
try {
await fs.unlink(localImagePath);
} catch { }
await sleep(500);
}
}
}
async function main() {
const server = getServerConfig();
// Infinite loop as requested
while (true) {
for (const KEYWORD of KEYWORDS) {
logger.info(`Starting Pinterest image conversion loop for keyword: "${KEYWORD}" (target ${TARGET_COUNT})`);
try {
const images = await collectImages(KEYWORD, TARGET_COUNT);
logger.info(`Collected ${images.length} image(s). Starting conversion...`);
await processImages(images, server);
logger.info('Iteration completed. Sleeping before next cycle...');
} catch (err) {
logger.error('Iteration failed:', err);
}
// brief pause between iterations
await sleep(10000);
}
}
}
process.on('unhandledRejection', (reason) => {
logger.error('UnhandledRejection:', reason);
});
process.on('uncaughtException', (err) => {
logger.error('UncaughtException:', err);
});
main().catch((e) => {
logger.error('Fatal error:', e);
});

0
src/imageconverter/pinterest_face_portrait.ts → src/tools/pinterest_face_portrait.ts
View File