diff --git a/.clinerules/generators.md b/.clinerules/generators.md new file mode 100644 index 0000000..7bbba3c --- /dev/null +++ b/.clinerules/generators.md @@ -0,0 +1,18 @@ +#Image and Video generation + +## Simple image generation with prompt +Use this file src\lib\image-generator.ts + +## Image generation with existing image +Use this file src\lib\image-generator-face.ts + +## Image converting +Use this file src\lib\image-converter.ts + +## Vide generation with static image +Use this ilfe src\lib\video-generator.ts + +Everything when generator need to use existing image you have to copy to input folder to the server and use only filename. +For example if +inputfolderFullpath = SERVER1_COMFY_OUTPUT_DIR.replece("output","input"); +Then copy image to the path, and pass only filename to generator function \ No newline at end of file diff --git a/.clinerules/llm.md b/.clinerules/llm.md new file mode 100644 index 0000000..a5adcf1 --- /dev/null +++ b/.clinerules/llm.md @@ -0,0 +1,17 @@ +# LLM API + +## LMstudio +Use this file src\lib\lmstudio.ts + +- async function callLmstudio(prompt: string): Promise { + for just run prompt +- async function callLMStudioAPIWithFile(imagePath: string, prompt: string): Promise { + for send file to llm + +## OpenAI +Use this file src\lib\openai.ts + +- async function callOpenAI(prompt: string): Promise + for just run prompt +- async function callOpenAIWithFile(imagePath: string, prompt: string): Promise + for send file to llm \ No newline at end of file diff --git a/src/comfyworkflows/cloth_extractor.json b/src/comfyworkflows/cloth_extractor.json new file mode 100644 index 0000000..cf31f70 --- /dev/null +++ b/src/comfyworkflows/cloth_extractor.json @@ -0,0 +1,207 @@ +{ + "1": { + "inputs": { + "unet_name": "qwen_image_edit_2509_fp8_e4m3fn.safetensors", + "weight_dtype": "default" + }, + "class_type": "UNETLoader", + "_meta": { + "title": "Load Diffusion Model" + } + }, + "2": { + "inputs": { + "clip_name": "qwen_2.5_vl_7b_fp8_scaled.safetensors", + "type": "qwen_image", + "device": "default" + }, + "class_type": "CLIPLoader", + "_meta": { + "title": "Load CLIP" + } + }, + "3": { + "inputs": { + "vae_name": "qwen_image_vae.safetensors" + }, + "class_type": "VAELoader", + "_meta": { + "title": "Load VAE" + } + }, + "4": { + "inputs": { + "lora_name": "Qwen-Image-Lightning-8steps-V2.0.safetensors", + "strength_model": 1, + "model": [ + "1", + 0 + ] + }, + "class_type": "LoraLoaderModelOnly", + "_meta": { + "title": "LoraLoaderModelOnly" + } + }, + "5": { + "inputs": { + "conditioning": [ + "11", + 0 + ] + }, + "class_type": "ConditioningZeroOut", + "_meta": { + "title": "ConditioningZeroOut" + } + }, + "7": { + "inputs": { + "seed": 1088883674457465, + "steps": 8, + "cfg": 1, + "sampler_name": "euler", + "scheduler": "beta", + "denoise": 1, + "model": [ + "66", + 0 + ], + "positive": [ + "11", + 0 + ], + "negative": [ + "5", + 0 + ], + "latent_image": [ + "11", + 6 + ] + }, + "class_type": "KSampler", + "_meta": { + "title": "KSampler" + } + }, + "8": { + "inputs": { + "samples": [ + "7", + 0 + ], + "vae": [ + "3", + 0 + ] + }, + "class_type": "VAEDecode", + "_meta": { + "title": "VAE Decode" + } + }, + "11": { + "inputs": { + "prompt": [ + "21", + 0 + ], + "enable_resize": true, + "enable_vl_resize": true, + "upscale_method": "lanczos", + "crop": "disabled", + "instruction": "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", + "clip": [ + "2", + 0 + ], + "vae": [ + "3", + 0 + ], + "image1": [ + "24", + 0 + ] + }, + "class_type": "TextEncodeQwenImageEditPlus_lrzjason", + "_meta": { + "title": "TextEncodeQwenImageEditPlus 小志Jason(xiaozhijason)" + } + }, + "20": { + "inputs": { + "filename_prefix": "qwenedit", + "images": [ + "8", + 0 + ] + }, + "class_type": "SaveImage", + "_meta": { + "title": "Save Image" + } + }, + "21": { + "inputs": { + "value": "extract the outfit onto a white background" + }, + "class_type": "PrimitiveStringMultiline", + "_meta": { + "title": "String (Multiline)" + } + }, + "24": { + "inputs": { + "measurement": "pixels", + "width": 720, + "height": 1280, + "fit": "contain", + "method": "nearest-exact", + "image": [ + "64", + 0 + ] + }, + "class_type": "Image Resize (rgthree)", + "_meta": { + "title": "Image Resize (rgthree)" + } + }, + "64": { + "inputs": { + "image": "3096293489212792_1758825204441_2.png" + }, + "class_type": "LoadImage", + "_meta": { + "title": "Load Image" + } + }, + "65": { + "inputs": { + "images": [ + "24", + 0 + ] + }, + "class_type": "PreviewImage", + "_meta": { + "title": "Preview Image" + } + }, + "66": { + "inputs": { + "lora_name": "extract-outfit_v3.safetensors", + "strength_model": 1, + "model": [ + "4", + 0 + ] + }, + "class_type": "LoraLoaderModelOnly", + "_meta": { + "title": "LoraLoaderModelOnly" + } + } +} \ No newline at end of file diff --git a/src/comfyworkflows/edit_image_2_qwen.json b/src/comfyworkflows/edit_image_2_qwen.json index ca6740d..a04dd60 100644 --- a/src/comfyworkflows/edit_image_2_qwen.json +++ b/src/comfyworkflows/edit_image_2_qwen.json @@ -57,7 +57,7 @@ }, "7": { "inputs": { - "seed": 135126386717887, + "seed": 838097333311955, "steps": 8, "cfg": 1, "sampler_name": "euler", @@ -158,7 +158,7 @@ }, "21": { "inputs": { - "value": "change the face and hairstyle of image1 to image2" + "value": "change camera angle to closeup face from image1, change background to light gray with faing gradient, change face angle to look at directry look at camera" }, "class_type": "PrimitiveStringMultiline", "_meta": { @@ -170,10 +170,10 @@ "measurement": "pixels", "width": 720, "height": 1280, - "fit": "pad", + "fit": "contain", "method": "nearest-exact", "image": [ - "63", + "64", 0 ] }, @@ -182,13 +182,25 @@ "title": "Image Resize (rgthree)" } }, - "63": { + "64": { "inputs": { - "image": "00d3c030856044bb99b804b4ef88f4be.jpg" + "image": "1337074888177434_1758776251440_2.png" }, "class_type": "LoadImage", "_meta": { "title": "Load Image" } + }, + "65": { + "inputs": { + "images": [ + "24", + 0 + ] + }, + "class_type": "PreviewImage", + "_meta": { + "title": "Preview Image" + } } } \ No newline at end of file diff --git a/src/lib/image-converter.ts b/src/lib/image-converter.ts index 0dc4bd8..7f0e37a 100644 --- a/src/lib/image-converter.ts +++ b/src/lib/image-converter.ts @@ -12,7 +12,7 @@ interface ImageSize { async function convertImage( prompt: string, - newFileName: string, + baseFileName: string, comfyBaseUrl: string, comfyOutputDir: string, size: ImageSize = { width: 720, height: 1280 } @@ -25,7 +25,7 @@ async function convertImage( workflow['21']['inputs']['value'] = prompt; workflow['24']['inputs']['width'] = size.width; workflow['24']['inputs']['height'] = size.height; - workflow['64']['inputs']['image'] = newFileName; + workflow['64']['inputs']['image'] = baseFileName; const response = await axios.post(`${COMFY_BASE_URL}/prompt`, { prompt: workflow }); const promptId = response.data.prompt_id; @@ -50,7 +50,7 @@ async function convertImage( fileStats.sort((a, b) => b.mtime.getTime() - a.mtime.getTime()); const latestFile = fileStats[0].file; - const newFilePath = path.resolve('./generated', newFileName); + const newFilePath = path.resolve('./generated', baseFileName); await fs.mkdir('./generated', { recursive: true }); @@ -66,4 +66,121 @@ async function convertImage( return newFilePath; } -export { convertImage }; + +async function convertImageWithFile( + prompt: string, + baseFileName: string, + secondFileName: string, + comfyBaseUrl: string, + comfyOutputDir: string, + size: ImageSize = { width: 720, height: 1280 } +): Promise { + const COMFY_BASE_URL = comfyBaseUrl.replace(/\/$/, ''); + const COMFY_OUTPUT_DIR = comfyOutputDir; + let workflow; + + workflow = JSON.parse(await fs.readFile('src/comfyworkflows/edit_image_2_qwen.json', 'utf-8')); + workflow['21']['inputs']['value'] = prompt; + workflow['24']['inputs']['width'] = size.width; + workflow['24']['inputs']['height'] = size.height; + workflow['64']['inputs']['image'] = baseFileName; + workflow['15']['inputs']['image'] = secondFileName; + + const response = await axios.post(`${COMFY_BASE_URL}/prompt`, { prompt: workflow }); + const promptId = response.data.prompt_id; + + let history; + do { + await new Promise(resolve => setTimeout(resolve, 1000)); + const historyResponse = await axios.get(`${COMFY_BASE_URL}/history/${promptId}`); + history = historyResponse.data[promptId]; + } while (!history || Object.keys(history.outputs).length === 0); + + const files = await fs.readdir(COMFY_OUTPUT_DIR!); + const generatedFiles = files.filter(file => file.startsWith('qwenedit')); + + const fileStats = await Promise.all( + generatedFiles.map(async (file) => { + const stat = await fs.stat(path.join(COMFY_OUTPUT_DIR!, file)); + return { file, mtime: stat.mtime }; + }) + ); + + fileStats.sort((a, b) => b.mtime.getTime() - a.mtime.getTime()); + + const latestFile = fileStats[0].file; + const newFilePath = path.resolve('./generated', baseFileName); + + await fs.mkdir('./generated', { recursive: true }); + + const sourcePath = path.join(COMFY_OUTPUT_DIR!, latestFile); + try { + await fs.unlink(newFilePath); + } catch (err) { + // ignore if not exists + } + + await fs.copyFile(sourcePath, newFilePath); + + return newFilePath; +} + + +export async function extractCloth( + prompt: string, + baseFileName: string, + comfyBaseUrl: string, + comfyOutputDir: string, + size: ImageSize = { width: 720, height: 1280 } +): Promise { + const COMFY_BASE_URL = comfyBaseUrl.replace(/\/$/, ''); + const COMFY_OUTPUT_DIR = comfyOutputDir; + let workflow; + + workflow = JSON.parse(await fs.readFile('src/comfyworkflows/cloth_extractor.json', 'utf-8')); + workflow['21']['inputs']['value'] = prompt; + workflow['24']['inputs']['width'] = size.width; + workflow['24']['inputs']['height'] = size.height; + workflow['64']['inputs']['image'] = baseFileName; + + const response = await axios.post(`${COMFY_BASE_URL}/prompt`, { prompt: workflow }); + const promptId = response.data.prompt_id; + + let history; + do { + await new Promise(resolve => setTimeout(resolve, 1000)); + const historyResponse = await axios.get(`${COMFY_BASE_URL}/history/${promptId}`); + history = historyResponse.data[promptId]; + } while (!history || Object.keys(history.outputs).length === 0); + + const files = await fs.readdir(COMFY_OUTPUT_DIR!); + const generatedFiles = files.filter(file => file.startsWith('qwenedit')); + + const fileStats = await Promise.all( + generatedFiles.map(async (file) => { + const stat = await fs.stat(path.join(COMFY_OUTPUT_DIR!, file)); + return { file, mtime: stat.mtime }; + }) + ); + + fileStats.sort((a, b) => b.mtime.getTime() - a.mtime.getTime()); + + const latestFile = fileStats[0].file; + const newFilePath = path.resolve('./generated', baseFileName); + + await fs.mkdir('./generated', { recursive: true }); + + const sourcePath = path.join(COMFY_OUTPUT_DIR!, latestFile); + try { + await fs.unlink(newFilePath); + } catch (err) { + // ignore if not exists + } + + await fs.copyFile(sourcePath, newFilePath); + + return newFilePath; +} + + +export { convertImage, convertImageWithFile }; diff --git a/src/tools/face_normalizer.ts b/src/tools/face_normalizer.ts new file mode 100644 index 0000000..0add2e3 --- /dev/null +++ b/src/tools/face_normalizer.ts @@ -0,0 +1,122 @@ +import * as fs from 'fs'; +import { promises as fsPromises } from 'fs'; +import * as path from 'path'; +import * as crypto from 'crypto'; +import dotenv from 'dotenv'; +import { callLMStudioAPIWithFile } from '../lib/lmstudio'; +import { generateImage } from '../lib/image-generator'; +import { convertImageWithFile } from '../lib/image-converter'; + +dotenv.config(); + +const inputDir = path.resolve('./input'); +const outputDir = path.resolve('./generated'); + +async function main() { + const comfyBaseUrl = process.env.SERVER1_COMFY_BASE_URL; + const comfyOutputDir = process.env.SERVER1_COMFY_OUTPUT_DIR; + + if (!comfyBaseUrl || !comfyOutputDir) { + throw new Error('SERVER1_COMFY_BASE_URL and SERVER1_COMFY_OUTPUT_DIR must be set in .env file'); + } + const comfyInputDir = comfyOutputDir.replace("output", "input"); + + // Ensure output directory exists + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + const files = fs.readdirSync(inputDir); + + for (const file of files) { + const imagePath = path.join(inputDir, file); + + // Calculate file hash + const fileBuffer = fs.readFileSync(imagePath); + const hashSum = crypto.createHash('sha256'); + hashSum.update(fileBuffer); + const hexHash = hashSum.digest('hex'); + + const genFileName = `${hexHash}_gen.png`; + const convertedFileName = `${hexHash}_converted.png`; + + const genFilePath = path.join(outputDir, genFileName); + const convertedFilePath = path.join(outputDir, convertedFileName); + + if (fs.existsSync(genFilePath) && fs.existsSync(convertedFilePath)) { + console.log(`Skipping ${file} as it has already been processed.`); + continue; + } + + console.log(`Processing ${imagePath}...`); + + // 1. Ask LLM to make a prompt for this photo + const llmPrompt = ` + Extract the following information from the attached image: face detail, hairstyle, eye color, hair color. + Output instruction: + { result: "face detail, hairstyle, eye color, hair color" } + `; + const extractedInfoJSON = await callLMStudioAPIWithFile(imagePath, llmPrompt); + + if (!extractedInfoJSON) { + console.error(`Could not get prompt from LLM for ${imagePath}`); + continue; + } + + const extractedInfo = extractedInfoJSON.result; + + console.log(`LLM response: ${extractedInfo}`); + + // 2. Generate photo using the prompt + const prefix = "Highly detailed face portrait of 20 years old girl, light gray background with faint gradient, look at camera in front"; + const finalPrompt = prefix + extractedInfo; + + console.log(`Generating image with prompt: ${finalPrompt}`); + + const generatedImagePath = await generateImage( + finalPrompt, + genFileName, + comfyBaseUrl, + comfyOutputDir, + 'flux' + ); + + if (generatedImagePath) { + console.log(`Generated image saved to: ${generatedImagePath}`); + + // 3. Convert the image + const generatedImageFileName = path.basename(generatedImagePath); + const originalImageFileName = file; + + const destGeneratedImagePath = path.join(comfyInputDir, generatedImageFileName); + const destOriginalImagePath = path.join(comfyInputDir, originalImageFileName); + + // Copy files to comfy input directory + await fsPromises.copyFile(generatedImagePath, destGeneratedImagePath); + await fsPromises.copyFile(imagePath, destOriginalImagePath); + console.log(`Copied ${generatedImageFileName} and ${originalImageFileName} to ${comfyInputDir}`); + + const conversionPrompt = "Replace the hairs of image1 with hairs of image2, change face of image1 with face of image2."; + const convertedResultPath = await convertImageWithFile( + conversionPrompt, + generatedImageFileName, + originalImageFileName, + comfyBaseUrl, + comfyOutputDir + ); + + if (convertedResultPath) { + // Rename the output file to the final converted name + await fsPromises.rename(convertedResultPath, convertedFilePath); + console.log(`Converted image saved to: ${convertedFilePath}`); + } else { + console.error(`Failed to convert image for ${originalImageFileName}`); + } + + } else { + console.error(`Failed to generate image for prompt: ${finalPrompt}`); + } + } +} + +main().catch(console.error); diff --git a/src/tools/pinterest_cloth_extraction.ts b/src/tools/pinterest_cloth_extraction.ts new file mode 100644 index 0000000..4d4e46f --- /dev/null +++ b/src/tools/pinterest_cloth_extraction.ts @@ -0,0 +1,173 @@ +import dotenv from 'dotenv'; +import * as fs from 'fs/promises'; +import * as path from 'path'; +import { logger } from '../lib/logger'; +import { getPinUrlFromPinterest, downloadImageFromPin } from '../lib/pinterest'; +import { convertImage, extractCloth } from '../lib/image-converter'; +dotenv.config(); + +const KEYWORDS = [ + 'teen skirt outfit', + 'teen casual cute outfit', + 'cute outfit', + 'summer dress', + 'elegant dress', + 'bohemian dress', + 'ball gown dress', + 'cosplay outfit', + 'vintage skirt outfit', + 'casual skirt outfit']; +const TARGET_COUNT = Number(process.env.IMAGE_COUNT || 20); +const PROMPT = + `Change pose of the person in the image1 to stantind in front of camera, with a smile, full body visible, wearing a fashionable outfit suitable for a casual day out. The background should be a white. Ensure the lighting is bright and natural, highlighting the details of the outfit and the person's features. + `; + +const PROMPT2 = + `Extract clothes from image1, put it on the white backgroud + `; + +type ServerCfg = { baseUrl: string; outputDir: string; inputDir: string }; + +function getServerConfig(): ServerCfg { + const candidates = [ + { baseUrl: process.env.SERVER1_COMFY_BASE_URL, outputDir: process.env.SERVER1_COMFY_OUTPUT_DIR }, + //{ baseUrl: process.env.SERVER2_COMFY_BASE_URL, outputDir: process.env.SERVER2_COMFY_OUTPUT_DIR }, + ].filter((s): s is { baseUrl: string; outputDir: string } => !!s.baseUrl && !!s.outputDir); + + if (candidates.length === 0) { + throw new Error( + 'No ComfyUI server configured. Please set SERVER1_COMFY_BASE_URL/OUTPUT_DIR or SERVER2_COMFY_BASE_URL/OUTPUT_DIR in .env' + ); + } + const chosen = candidates[0]; + const inputDir = chosen.outputDir.replace('output', 'input'); + return { ...chosen, inputDir }; +} + +function sleep(ms: number) { + return new Promise((res) => setTimeout(res, ms)); +} + +async function ensureDir(p: string) { + try { + await fs.mkdir(p, { recursive: true }); + } catch { + // ignore + } +} + +async function collectImages(keyword: string, total: number): Promise { + const results: string[] = []; + // ensure local download dir exists (pinterest.ts also ensures it, but harmless here) + await ensureDir(path.join(process.cwd(), 'download')); + + while (results.length < total) { + try { + const pinUrl = await getPinUrlFromPinterest(keyword); + if (!pinUrl) { + logger.warn('No pin URL found, retrying...'); + await sleep(1500); + continue; + } + const remaining = total - results.length; + // attempt to grab up to 5 per pin to reduce churn + const batchTarget = Math.min(5, remaining); + const imgs = await downloadImageFromPin(pinUrl, batchTarget); + if (imgs && imgs.length > 0) { + results.push(...imgs); + logger.info(`Downloaded ${imgs.length} image(s) from ${pinUrl}.Progress: ${results.length}/${total}`); + } else { + logger.warn(`Pin yielded no downloadable images: ${pinUrl}`); + } + await sleep(1000 + Math.random() * 1000); + } catch (err) { + logger.error('Error while collecting images:', err); + await sleep(2000); + } + } + return results.slice(0, total); +} + +async function processImages(imagePaths: string[], server: ServerCfg) { + await ensureDir(server.inputDir); + + for (const localImagePath of imagePaths) { + const baseName = path.basename(localImagePath); + const serverInputPath = path.join(server.inputDir, baseName); + + try { + // copy source image into ComfyUI input dir so workflow node can find it by filename + await fs.copyFile(localImagePath, serverInputPath); + logger.info(`Copied ${localImagePath} -> ${serverInputPath}`); + + // Run conversion (sequential to avoid output race conditions) + const generatedPath1 = await convertImage( + PROMPT, + baseName, + server.baseUrl, + server.outputDir, + { width: 720, height: 1280 } // portrait + ); + + await fs.copyFile(generatedPath1, serverInputPath); + const baseName2 = path.basename(localImagePath); + + const generatedPath2 = await extractCloth( + PROMPT2, + baseName2, + server.baseUrl, + server.outputDir, + { width: 720, height: 1280 } // portrait + ); + + logger.info(`Generated image: ${generatedPath2}`); + } catch (err) { + logger.error(`Failed to convert ${localImagePath}:`, err); + } finally { + // cleanup both server input copy and local downloaded file + try { + await fs.unlink(serverInputPath); + } catch { } + try { + await fs.unlink(localImagePath); + } catch { } + await sleep(500); + } + } +} + +async function main() { + const server = getServerConfig(); + + // Infinite loop as requested + while (true) { + + for (const KEYWORD of KEYWORDS) { + logger.info(`Starting Pinterest image conversion loop for keyword: "${KEYWORD}" (target ${TARGET_COUNT})`); + try { + const images = await collectImages(KEYWORD, TARGET_COUNT); + logger.info(`Collected ${images.length} image(s). Starting conversion...`); + await processImages(images, server); + logger.info('Iteration completed. Sleeping before next cycle...'); + } catch (err) { + logger.error('Iteration failed:', err); + } + + // brief pause between iterations + await sleep(10000); + } + + } +} + +process.on('unhandledRejection', (reason) => { + logger.error('UnhandledRejection:', reason); +}); + +process.on('uncaughtException', (err) => { + logger.error('UncaughtException:', err); +}); + +main().catch((e) => { + logger.error('Fatal error:', e); +}); diff --git a/src/imageconverter/pinterest_face_portrait.ts b/src/tools/pinterest_face_portrait.ts similarity index 100% rename from src/imageconverter/pinterest_face_portrait.ts rename to src/tools/pinterest_face_portrait.ts