save changes
This commit is contained in:
581
src/comfyworkflows/prototyping_style_flux_wan22.json
Normal file
581
src/comfyworkflows/prototyping_style_flux_wan22.json
Normal file
@ -0,0 +1,581 @@
|
||||
{
|
||||
"70": {
|
||||
"inputs": {
|
||||
"text": "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
|
||||
"clip": [
|
||||
"82",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": {
|
||||
"title": "CLIP Text Encode (Negative Prompt)"
|
||||
}
|
||||
},
|
||||
"73": {
|
||||
"inputs": {
|
||||
"vae_name": "wan_2.1_vae.safetensors"
|
||||
},
|
||||
"class_type": "VAELoader",
|
||||
"_meta": {
|
||||
"title": "Load VAE"
|
||||
}
|
||||
},
|
||||
"76": {
|
||||
"inputs": {
|
||||
"enable_fp16_accumulation": true,
|
||||
"model": [
|
||||
"77",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ModelPatchTorchSettings",
|
||||
"_meta": {
|
||||
"title": "Model Patch Torch Settings"
|
||||
}
|
||||
},
|
||||
"77": {
|
||||
"inputs": {
|
||||
"sage_attention": "sageattn_qk_int8_pv_fp8_cuda++",
|
||||
"model": [
|
||||
"83",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "PathchSageAttentionKJ",
|
||||
"_meta": {
|
||||
"title": "Patch Sage Attention KJ"
|
||||
}
|
||||
},
|
||||
"78": {
|
||||
"inputs": {
|
||||
"sage_attention": "sageattn_qk_int8_pv_fp8_cuda++",
|
||||
"model": [
|
||||
"84",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "PathchSageAttentionKJ",
|
||||
"_meta": {
|
||||
"title": "Patch Sage Attention KJ"
|
||||
}
|
||||
},
|
||||
"79": {
|
||||
"inputs": {
|
||||
"enable_fp16_accumulation": true,
|
||||
"model": [
|
||||
"78",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ModelPatchTorchSettings",
|
||||
"_meta": {
|
||||
"title": "Model Patch Torch Settings"
|
||||
}
|
||||
},
|
||||
"80": {
|
||||
"inputs": {
|
||||
"shift": 8.000000000000002,
|
||||
"model": [
|
||||
"89",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ModelSamplingSD3",
|
||||
"_meta": {
|
||||
"title": "ModelSamplingSD3"
|
||||
}
|
||||
},
|
||||
"81": {
|
||||
"inputs": {
|
||||
"shift": 8.000000000000002,
|
||||
"model": [
|
||||
"90",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ModelSamplingSD3",
|
||||
"_meta": {
|
||||
"title": "ModelSamplingSD3"
|
||||
}
|
||||
},
|
||||
"82": {
|
||||
"inputs": {
|
||||
"clip_name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors",
|
||||
"type": "wan",
|
||||
"device": "cpu"
|
||||
},
|
||||
"class_type": "CLIPLoader",
|
||||
"_meta": {
|
||||
"title": "Load CLIP"
|
||||
}
|
||||
},
|
||||
"83": {
|
||||
"inputs": {
|
||||
"unet_name": "wan2.2_i2v_high_noise_14B_Q4_K_S.gguf"
|
||||
},
|
||||
"class_type": "UnetLoaderGGUF",
|
||||
"_meta": {
|
||||
"title": "Unet Loader (GGUF)"
|
||||
}
|
||||
},
|
||||
"84": {
|
||||
"inputs": {
|
||||
"unet_name": "wan2.2_i2v_low_noise_14B_Q4_K_S.gguf"
|
||||
},
|
||||
"class_type": "UnetLoaderGGUF",
|
||||
"_meta": {
|
||||
"title": "Unet Loader (GGUF)"
|
||||
}
|
||||
},
|
||||
"85": {
|
||||
"inputs": {
|
||||
"frame_rate": 32,
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "STYLEDVIDEOMAKER",
|
||||
"format": "video/h264-mp4",
|
||||
"pix_fmt": "yuv420p",
|
||||
"crf": 19,
|
||||
"save_metadata": true,
|
||||
"trim_to_audio": false,
|
||||
"pingpong": false,
|
||||
"save_output": true,
|
||||
"images": [
|
||||
"88",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "VHS_VideoCombine",
|
||||
"_meta": {
|
||||
"title": "Video Combine 🎥🅥🅗🅢"
|
||||
}
|
||||
},
|
||||
"86": {
|
||||
"inputs": {
|
||||
"upscale_model": "4x-UltraSharp.pth",
|
||||
"mode": "rescale",
|
||||
"rescale_factor": 2.0000000000000004,
|
||||
"resize_width": 832,
|
||||
"resampling_method": "lanczos",
|
||||
"supersample": "true",
|
||||
"rounding_modulus": 8
|
||||
},
|
||||
"class_type": "CR Upscale Image",
|
||||
"_meta": {
|
||||
"title": "🔍 CR Upscale Image"
|
||||
}
|
||||
},
|
||||
"87": {
|
||||
"inputs": {
|
||||
"samples": [
|
||||
"92",
|
||||
0
|
||||
],
|
||||
"vae": [
|
||||
"73",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "VAEDecode",
|
||||
"_meta": {
|
||||
"title": "VAE Decode"
|
||||
}
|
||||
},
|
||||
"88": {
|
||||
"inputs": {
|
||||
"ckpt_name": "rife49.pth",
|
||||
"clear_cache_after_n_frames": 10,
|
||||
"multiplier": 2,
|
||||
"fast_mode": true,
|
||||
"ensemble": true,
|
||||
"scale_factor": 1,
|
||||
"frames": [
|
||||
"98",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "RIFE VFI",
|
||||
"_meta": {
|
||||
"title": "RIFE VFI (recommend rife47 and rife49)"
|
||||
}
|
||||
},
|
||||
"89": {
|
||||
"inputs": {
|
||||
"lora_name": "Wan21_I2V_14B_lightx2v_cfg_step_distill_lora_rank64.safetensors",
|
||||
"strength_model": 3.0000000000000004,
|
||||
"model": [
|
||||
"76",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "LoraLoaderModelOnly",
|
||||
"_meta": {
|
||||
"title": "LoraLoaderModelOnly"
|
||||
}
|
||||
},
|
||||
"90": {
|
||||
"inputs": {
|
||||
"lora_name": "Wan21_T2V_14B_lightx2v_cfg_step_distill_lora_rank64.safetensors",
|
||||
"strength_model": 1.5000000000000002,
|
||||
"model": [
|
||||
"79",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "LoraLoaderModelOnly",
|
||||
"_meta": {
|
||||
"title": "LoraLoaderModelOnly"
|
||||
}
|
||||
},
|
||||
"91": {
|
||||
"inputs": {
|
||||
"add_noise": "enable",
|
||||
"noise_seed": 452107028428,
|
||||
"steps": 6,
|
||||
"cfg": 1,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"start_at_step": 0,
|
||||
"end_at_step": 3,
|
||||
"return_with_leftover_noise": "enable",
|
||||
"model": [
|
||||
"80",
|
||||
0
|
||||
],
|
||||
"positive": [
|
||||
"96",
|
||||
0
|
||||
],
|
||||
"negative": [
|
||||
"96",
|
||||
1
|
||||
],
|
||||
"latent_image": [
|
||||
"96",
|
||||
2
|
||||
]
|
||||
},
|
||||
"class_type": "KSamplerAdvanced",
|
||||
"_meta": {
|
||||
"title": "KSampler (Advanced)"
|
||||
}
|
||||
},
|
||||
"92": {
|
||||
"inputs": {
|
||||
"add_noise": "disable",
|
||||
"noise_seed": 0,
|
||||
"steps": 6,
|
||||
"cfg": 1,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"start_at_step": 3,
|
||||
"end_at_step": 10000,
|
||||
"return_with_leftover_noise": "disable",
|
||||
"model": [
|
||||
"81",
|
||||
0
|
||||
],
|
||||
"positive": [
|
||||
"96",
|
||||
0
|
||||
],
|
||||
"negative": [
|
||||
"96",
|
||||
1
|
||||
],
|
||||
"latent_image": [
|
||||
"91",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "KSamplerAdvanced",
|
||||
"_meta": {
|
||||
"title": "KSampler (Advanced)"
|
||||
}
|
||||
},
|
||||
"93": {
|
||||
"inputs": {
|
||||
"model": "4xNomos2_otf_esrgan",
|
||||
"precision": "fp16"
|
||||
},
|
||||
"class_type": "LoadUpscalerTensorrtModel",
|
||||
"_meta": {
|
||||
"title": "Load Upscale Tensorrt Model"
|
||||
}
|
||||
},
|
||||
"95": {
|
||||
"inputs": {
|
||||
"text": "A luminous ballerina in mid-performance, illustrated in glowing white strokes against a deep black background. She balances gracefully en pointe on one foot, arms extended in a fluid pose, her tutu radiating light and motion like ethereal fabric made of starlight. Her hair flows upward, sketched in swirling white lines, blending with scattered glowing stars around her. The reflection of her figure shimmers on a dark water surface below, surrounded by circular ripples of light. The style is dreamy, abstract, and expressive, combining sketch-like brush strokes with glowing energy lines. High contrast, elegant, surreal ballet illustration.",
|
||||
"clip": [
|
||||
"82",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": {
|
||||
"title": "CLIP Text Encode (Positive Prompt)"
|
||||
}
|
||||
},
|
||||
"96": {
|
||||
"inputs": {
|
||||
"width": [
|
||||
"97",
|
||||
1
|
||||
],
|
||||
"height": [
|
||||
"97",
|
||||
2
|
||||
],
|
||||
"length": 121,
|
||||
"batch_size": 1,
|
||||
"positive": [
|
||||
"95",
|
||||
0
|
||||
],
|
||||
"negative": [
|
||||
"70",
|
||||
0
|
||||
],
|
||||
"vae": [
|
||||
"73",
|
||||
0
|
||||
],
|
||||
"start_image": [
|
||||
"97",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "WanImageToVideo",
|
||||
"_meta": {
|
||||
"title": "WanImageToVideo"
|
||||
}
|
||||
},
|
||||
"97": {
|
||||
"inputs": {
|
||||
"width": 320,
|
||||
"height": 640,
|
||||
"upscale_method": "lanczos",
|
||||
"keep_proportion": "crop",
|
||||
"pad_color": "0, 0, 0",
|
||||
"crop_position": "center",
|
||||
"divisible_by": 16,
|
||||
"device": "cpu",
|
||||
"image": [
|
||||
"103",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ImageResizeKJv2",
|
||||
"_meta": {
|
||||
"title": "Resize Image v2"
|
||||
}
|
||||
},
|
||||
"98": {
|
||||
"inputs": {
|
||||
"resize_to": "4k",
|
||||
"images": [
|
||||
"87",
|
||||
0
|
||||
],
|
||||
"upscaler_trt_model": [
|
||||
"93",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "UpscalerTensorrt",
|
||||
"_meta": {
|
||||
"title": "Upscaler Tensorrt ⚡"
|
||||
}
|
||||
},
|
||||
"99": {
|
||||
"inputs": {
|
||||
"filename_prefix": "STYLEDVIDEOMAKER",
|
||||
"images": [
|
||||
"103",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "SaveImage",
|
||||
"_meta": {
|
||||
"title": "Save Image"
|
||||
}
|
||||
},
|
||||
"100": {
|
||||
"inputs": {
|
||||
"clip_name1": "clip_l.safetensors",
|
||||
"clip_name2": "t5xxl_fp16.safetensors",
|
||||
"type": "flux",
|
||||
"device": "default"
|
||||
},
|
||||
"class_type": "DualCLIPLoader",
|
||||
"_meta": {
|
||||
"title": "DualCLIPLoader"
|
||||
}
|
||||
},
|
||||
"101": {
|
||||
"inputs": {
|
||||
"vae_name": "ae.safetensors"
|
||||
},
|
||||
"class_type": "VAELoader",
|
||||
"_meta": {
|
||||
"title": "Load VAE"
|
||||
}
|
||||
},
|
||||
"102": {
|
||||
"inputs": {
|
||||
"conditioning": [
|
||||
"105",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ConditioningZeroOut",
|
||||
"_meta": {
|
||||
"title": "ConditioningZeroOut"
|
||||
}
|
||||
},
|
||||
"103": {
|
||||
"inputs": {
|
||||
"samples": [
|
||||
"106",
|
||||
0
|
||||
],
|
||||
"vae": [
|
||||
"101",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "VAEDecode",
|
||||
"_meta": {
|
||||
"title": "VAE Decode"
|
||||
}
|
||||
},
|
||||
"104": {
|
||||
"inputs": {
|
||||
"width": 320,
|
||||
"height": 640,
|
||||
"batch_size": 1
|
||||
},
|
||||
"class_type": "EmptySD3LatentImage",
|
||||
"_meta": {
|
||||
"title": "EmptySD3LatentImage"
|
||||
}
|
||||
},
|
||||
"105": {
|
||||
"inputs": {
|
||||
"text": "A luminous ballerina in mid-performance, illustrated in glowing white strokes against a deep black background. She balances gracefully en pointe on one foot, arms extended in a fluid pose, her tutu radiating light and motion like ethereal fabric made of starlight. Her hair flows upward, sketched in swirling white lines, blending with scattered glowing stars around her. The reflection of her figure shimmers on a dark water surface below, surrounded by circular ripples of light. The style is dreamy, abstract, and expressive, combining sketch-like brush strokes with glowing energy lines. High contrast, elegant, surreal ballet illustration.",
|
||||
"clip": [
|
||||
"100",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": {
|
||||
"title": "CLIP Text Encode (Prompt)"
|
||||
}
|
||||
},
|
||||
"106": {
|
||||
"inputs": {
|
||||
"seed": 84283616550942,
|
||||
"steps": 20,
|
||||
"cfg": 1,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"denoise": 1,
|
||||
"model": [
|
||||
"107",
|
||||
0
|
||||
],
|
||||
"positive": [
|
||||
"111",
|
||||
0
|
||||
],
|
||||
"negative": [
|
||||
"102",
|
||||
0
|
||||
],
|
||||
"latent_image": [
|
||||
"104",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "KSampler",
|
||||
"_meta": {
|
||||
"title": "KSampler"
|
||||
}
|
||||
},
|
||||
"107": {
|
||||
"inputs": {
|
||||
"unet_name": "flux1-krea-dev_fp8_scaled.safetensors",
|
||||
"weight_dtype": "default"
|
||||
},
|
||||
"class_type": "UNETLoader",
|
||||
"_meta": {
|
||||
"title": "Load Diffusion Model"
|
||||
}
|
||||
},
|
||||
"110": {
|
||||
"inputs": {
|
||||
"style_model_name": "flux1-redux-dev.safetensors"
|
||||
},
|
||||
"class_type": "StyleModelLoader",
|
||||
"_meta": {
|
||||
"title": "Load Style Model"
|
||||
}
|
||||
},
|
||||
"111": {
|
||||
"inputs": {
|
||||
"strength": 0.15,
|
||||
"conditioning": [
|
||||
"105",
|
||||
0
|
||||
],
|
||||
"style_model": [
|
||||
"110",
|
||||
0
|
||||
],
|
||||
"clip_vision_output": [
|
||||
"112",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ApplyStyleModelAdjust",
|
||||
"_meta": {
|
||||
"title": "Apply Style Model (Adjusted)"
|
||||
}
|
||||
},
|
||||
"112": {
|
||||
"inputs": {
|
||||
"crop": "center",
|
||||
"clip_vision": [
|
||||
"113",
|
||||
0
|
||||
],
|
||||
"image": [
|
||||
"114",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "CLIPVisionEncode",
|
||||
"_meta": {
|
||||
"title": "CLIP Vision Encode"
|
||||
}
|
||||
},
|
||||
"113": {
|
||||
"inputs": {
|
||||
"clip_name": "sigclip_vision_patch14_384.safetensors"
|
||||
},
|
||||
"class_type": "CLIPVisionLoader",
|
||||
"_meta": {
|
||||
"title": "Load CLIP Vision"
|
||||
}
|
||||
},
|
||||
"114": {
|
||||
"inputs": {
|
||||
"image": "440fb3cb2bcc993bdc7da34986a7135d.jpg"
|
||||
},
|
||||
"class_type": "LoadImage",
|
||||
"_meta": {
|
||||
"title": "Load Image"
|
||||
}
|
||||
}
|
||||
}
|
||||
545
src/comfyworkflows/prototyping_style_flux_wan225b.json
Normal file
545
src/comfyworkflows/prototyping_style_flux_wan225b.json
Normal file
@ -0,0 +1,545 @@
|
||||
{
|
||||
"70": {
|
||||
"inputs": {
|
||||
"text": "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
|
||||
"clip": [
|
||||
"82",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": {
|
||||
"title": "CLIP Text Encode (Negative Prompt)"
|
||||
}
|
||||
},
|
||||
"73": {
|
||||
"inputs": {
|
||||
"vae_name": "wan_2.1_vae.safetensors"
|
||||
},
|
||||
"class_type": "VAELoader",
|
||||
"_meta": {
|
||||
"title": "Load VAE"
|
||||
}
|
||||
},
|
||||
"76": {
|
||||
"inputs": {
|
||||
"enable_fp16_accumulation": true,
|
||||
"model": [
|
||||
"77",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ModelPatchTorchSettings",
|
||||
"_meta": {
|
||||
"title": "Model Patch Torch Settings"
|
||||
}
|
||||
},
|
||||
"77": {
|
||||
"inputs": {
|
||||
"sage_attention": "sageattn_qk_int8_pv_fp8_cuda++",
|
||||
"model": [
|
||||
"83",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "PathchSageAttentionKJ",
|
||||
"_meta": {
|
||||
"title": "Patch Sage Attention KJ"
|
||||
}
|
||||
},
|
||||
"78": {
|
||||
"inputs": {
|
||||
"sage_attention": "sageattn_qk_int8_pv_fp8_cuda++",
|
||||
"model": [
|
||||
"84",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "PathchSageAttentionKJ",
|
||||
"_meta": {
|
||||
"title": "Patch Sage Attention KJ"
|
||||
}
|
||||
},
|
||||
"79": {
|
||||
"inputs": {
|
||||
"enable_fp16_accumulation": true,
|
||||
"model": [
|
||||
"78",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ModelPatchTorchSettings",
|
||||
"_meta": {
|
||||
"title": "Model Patch Torch Settings"
|
||||
}
|
||||
},
|
||||
"80": {
|
||||
"inputs": {
|
||||
"shift": 8.000000000000002,
|
||||
"model": [
|
||||
"89",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ModelSamplingSD3",
|
||||
"_meta": {
|
||||
"title": "ModelSamplingSD3"
|
||||
}
|
||||
},
|
||||
"81": {
|
||||
"inputs": {
|
||||
"shift": 8.000000000000002,
|
||||
"model": [
|
||||
"90",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ModelSamplingSD3",
|
||||
"_meta": {
|
||||
"title": "ModelSamplingSD3"
|
||||
}
|
||||
},
|
||||
"82": {
|
||||
"inputs": {
|
||||
"clip_name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors",
|
||||
"type": "wan",
|
||||
"device": "cpu"
|
||||
},
|
||||
"class_type": "CLIPLoader",
|
||||
"_meta": {
|
||||
"title": "Load CLIP"
|
||||
}
|
||||
},
|
||||
"83": {
|
||||
"inputs": {
|
||||
"unet_name": "wan2.2_i2v_high_noise_14B_Q4_K_S.gguf"
|
||||
},
|
||||
"class_type": "UnetLoaderGGUF",
|
||||
"_meta": {
|
||||
"title": "Unet Loader (GGUF)"
|
||||
}
|
||||
},
|
||||
"84": {
|
||||
"inputs": {
|
||||
"unet_name": "wan2.2_i2v_low_noise_14B_Q4_K_S.gguf"
|
||||
},
|
||||
"class_type": "UnetLoaderGGUF",
|
||||
"_meta": {
|
||||
"title": "Unet Loader (GGUF)"
|
||||
}
|
||||
},
|
||||
"85": {
|
||||
"inputs": {
|
||||
"frame_rate": 32,
|
||||
"loop_count": 0,
|
||||
"filename_prefix": "STYLEDVIDEOMAKER",
|
||||
"format": "video/h264-mp4",
|
||||
"pix_fmt": "yuv420p",
|
||||
"crf": 19,
|
||||
"save_metadata": true,
|
||||
"trim_to_audio": false,
|
||||
"pingpong": false,
|
||||
"save_output": true,
|
||||
"images": [
|
||||
"87",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "VHS_VideoCombine",
|
||||
"_meta": {
|
||||
"title": "Video Combine 🎥🅥🅗🅢"
|
||||
}
|
||||
},
|
||||
"86": {
|
||||
"inputs": {
|
||||
"upscale_model": "4x-UltraSharp.pth",
|
||||
"mode": "rescale",
|
||||
"rescale_factor": 2.0000000000000004,
|
||||
"resize_width": 832,
|
||||
"resampling_method": "lanczos",
|
||||
"supersample": "true",
|
||||
"rounding_modulus": 8
|
||||
},
|
||||
"class_type": "CR Upscale Image",
|
||||
"_meta": {
|
||||
"title": "🔍 CR Upscale Image"
|
||||
}
|
||||
},
|
||||
"87": {
|
||||
"inputs": {
|
||||
"samples": [
|
||||
"92",
|
||||
0
|
||||
],
|
||||
"vae": [
|
||||
"73",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "VAEDecode",
|
||||
"_meta": {
|
||||
"title": "VAE Decode"
|
||||
}
|
||||
},
|
||||
"89": {
|
||||
"inputs": {
|
||||
"lora_name": "Wan21_I2V_14B_lightx2v_cfg_step_distill_lora_rank64.safetensors",
|
||||
"strength_model": 3.0000000000000004,
|
||||
"model": [
|
||||
"76",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "LoraLoaderModelOnly",
|
||||
"_meta": {
|
||||
"title": "LoraLoaderModelOnly"
|
||||
}
|
||||
},
|
||||
"90": {
|
||||
"inputs": {
|
||||
"lora_name": "Wan21_T2V_14B_lightx2v_cfg_step_distill_lora_rank64.safetensors",
|
||||
"strength_model": 1.5000000000000002,
|
||||
"model": [
|
||||
"79",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "LoraLoaderModelOnly",
|
||||
"_meta": {
|
||||
"title": "LoraLoaderModelOnly"
|
||||
}
|
||||
},
|
||||
"91": {
|
||||
"inputs": {
|
||||
"add_noise": "enable",
|
||||
"noise_seed": 452107028428,
|
||||
"steps": 6,
|
||||
"cfg": 1,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"start_at_step": 0,
|
||||
"end_at_step": 3,
|
||||
"return_with_leftover_noise": "enable",
|
||||
"model": [
|
||||
"80",
|
||||
0
|
||||
],
|
||||
"positive": [
|
||||
"96",
|
||||
0
|
||||
],
|
||||
"negative": [
|
||||
"96",
|
||||
1
|
||||
],
|
||||
"latent_image": [
|
||||
"96",
|
||||
2
|
||||
]
|
||||
},
|
||||
"class_type": "KSamplerAdvanced",
|
||||
"_meta": {
|
||||
"title": "KSampler (Advanced)"
|
||||
}
|
||||
},
|
||||
"92": {
|
||||
"inputs": {
|
||||
"add_noise": "disable",
|
||||
"noise_seed": 0,
|
||||
"steps": 6,
|
||||
"cfg": 1,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"start_at_step": 3,
|
||||
"end_at_step": 10000,
|
||||
"return_with_leftover_noise": "disable",
|
||||
"model": [
|
||||
"81",
|
||||
0
|
||||
],
|
||||
"positive": [
|
||||
"96",
|
||||
0
|
||||
],
|
||||
"negative": [
|
||||
"96",
|
||||
1
|
||||
],
|
||||
"latent_image": [
|
||||
"91",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "KSamplerAdvanced",
|
||||
"_meta": {
|
||||
"title": "KSampler (Advanced)"
|
||||
}
|
||||
},
|
||||
"94": {
|
||||
"inputs": {
|
||||
"image": "ComfyUI_00036_.png"
|
||||
},
|
||||
"class_type": "LoadImage",
|
||||
"_meta": {
|
||||
"title": "Load Image"
|
||||
}
|
||||
},
|
||||
"95": {
|
||||
"inputs": {
|
||||
"text": "A luminous ballerina in mid-performance, illustrated in glowing white strokes against a deep black background. She balances gracefully en pointe on one foot, arms extended in a fluid pose, her tutu radiating light and motion like ethereal fabric made of starlight. Her hair flows upward, sketched in swirling white lines, blending with scattered glowing stars around her. The reflection of her figure shimmers on a dark water surface below, surrounded by circular ripples of light. The style is dreamy, abstract, and expressive, combining sketch-like brush strokes with glowing energy lines. High contrast, elegant, surreal ballet illustration.",
|
||||
"clip": [
|
||||
"82",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": {
|
||||
"title": "CLIP Text Encode (Positive Prompt)"
|
||||
}
|
||||
},
|
||||
"96": {
|
||||
"inputs": {
|
||||
"width": [
|
||||
"97",
|
||||
1
|
||||
],
|
||||
"height": [
|
||||
"97",
|
||||
2
|
||||
],
|
||||
"length": 89,
|
||||
"batch_size": 1,
|
||||
"positive": [
|
||||
"95",
|
||||
0
|
||||
],
|
||||
"negative": [
|
||||
"70",
|
||||
0
|
||||
],
|
||||
"vae": [
|
||||
"73",
|
||||
0
|
||||
],
|
||||
"start_image": [
|
||||
"97",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "WanImageToVideo",
|
||||
"_meta": {
|
||||
"title": "WanImageToVideo"
|
||||
}
|
||||
},
|
||||
"97": {
|
||||
"inputs": {
|
||||
"width": 320,
|
||||
"height": 640,
|
||||
"upscale_method": "lanczos",
|
||||
"keep_proportion": "crop",
|
||||
"pad_color": "0, 0, 0",
|
||||
"crop_position": "center",
|
||||
"divisible_by": 16,
|
||||
"device": "cpu",
|
||||
"image": [
|
||||
"103",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ImageResizeKJv2",
|
||||
"_meta": {
|
||||
"title": "Resize Image v2"
|
||||
}
|
||||
},
|
||||
"99": {
|
||||
"inputs": {
|
||||
"filename_prefix": "STYLEDVIDEOMAKER",
|
||||
"images": [
|
||||
"103",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "SaveImage",
|
||||
"_meta": {
|
||||
"title": "Save Image"
|
||||
}
|
||||
},
|
||||
"100": {
|
||||
"inputs": {
|
||||
"clip_name1": "clip_l.safetensors",
|
||||
"clip_name2": "t5xxl_fp16.safetensors",
|
||||
"type": "flux",
|
||||
"device": "default"
|
||||
},
|
||||
"class_type": "DualCLIPLoader",
|
||||
"_meta": {
|
||||
"title": "DualCLIPLoader"
|
||||
}
|
||||
},
|
||||
"101": {
|
||||
"inputs": {
|
||||
"vae_name": "ae.safetensors"
|
||||
},
|
||||
"class_type": "VAELoader",
|
||||
"_meta": {
|
||||
"title": "Load VAE"
|
||||
}
|
||||
},
|
||||
"102": {
|
||||
"inputs": {
|
||||
"conditioning": [
|
||||
"105",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ConditioningZeroOut",
|
||||
"_meta": {
|
||||
"title": "ConditioningZeroOut"
|
||||
}
|
||||
},
|
||||
"103": {
|
||||
"inputs": {
|
||||
"samples": [
|
||||
"106",
|
||||
0
|
||||
],
|
||||
"vae": [
|
||||
"101",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "VAEDecode",
|
||||
"_meta": {
|
||||
"title": "VAE Decode"
|
||||
}
|
||||
},
|
||||
"104": {
|
||||
"inputs": {
|
||||
"width": 320,
|
||||
"height": 640,
|
||||
"batch_size": 1
|
||||
},
|
||||
"class_type": "EmptySD3LatentImage",
|
||||
"_meta": {
|
||||
"title": "EmptySD3LatentImage"
|
||||
}
|
||||
},
|
||||
"105": {
|
||||
"inputs": {
|
||||
"text": "A luminous ballerina in mid-performance, illustrated in glowing white strokes against a deep black background. She balances gracefully en pointe on one foot, arms extended in a fluid pose, her tutu radiating light and motion like ethereal fabric made of starlight. Her hair flows upward, sketched in swirling white lines, blending with scattered glowing stars around her. The reflection of her figure shimmers on a dark water surface below, surrounded by circular ripples of light. The style is dreamy, abstract, and expressive, combining sketch-like brush strokes with glowing energy lines. High contrast, elegant, surreal ballet illustration.",
|
||||
"clip": [
|
||||
"100",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "CLIPTextEncode",
|
||||
"_meta": {
|
||||
"title": "CLIP Text Encode (Prompt)"
|
||||
}
|
||||
},
|
||||
"106": {
|
||||
"inputs": {
|
||||
"seed": 84283616550942,
|
||||
"steps": 20,
|
||||
"cfg": 1,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "simple",
|
||||
"denoise": 1,
|
||||
"model": [
|
||||
"107",
|
||||
0
|
||||
],
|
||||
"positive": [
|
||||
"111",
|
||||
0
|
||||
],
|
||||
"negative": [
|
||||
"102",
|
||||
0
|
||||
],
|
||||
"latent_image": [
|
||||
"104",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "KSampler",
|
||||
"_meta": {
|
||||
"title": "KSampler"
|
||||
}
|
||||
},
|
||||
"107": {
|
||||
"inputs": {
|
||||
"unet_name": "flux1-krea-dev_fp8_scaled.safetensors",
|
||||
"weight_dtype": "default"
|
||||
},
|
||||
"class_type": "UNETLoader",
|
||||
"_meta": {
|
||||
"title": "Load Diffusion Model"
|
||||
}
|
||||
},
|
||||
"110": {
|
||||
"inputs": {
|
||||
"style_model_name": "flux1-redux-dev.safetensors"
|
||||
},
|
||||
"class_type": "StyleModelLoader",
|
||||
"_meta": {
|
||||
"title": "Load Style Model"
|
||||
}
|
||||
},
|
||||
"111": {
|
||||
"inputs": {
|
||||
"strength": 0.15,
|
||||
"conditioning": [
|
||||
"105",
|
||||
0
|
||||
],
|
||||
"style_model": [
|
||||
"110",
|
||||
0
|
||||
],
|
||||
"clip_vision_output": [
|
||||
"112",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "ApplyStyleModelAdjust",
|
||||
"_meta": {
|
||||
"title": "Apply Style Model (Adjusted)"
|
||||
}
|
||||
},
|
||||
"112": {
|
||||
"inputs": {
|
||||
"crop": "center",
|
||||
"clip_vision": [
|
||||
"113",
|
||||
0
|
||||
],
|
||||
"image": [
|
||||
"114",
|
||||
0
|
||||
]
|
||||
},
|
||||
"class_type": "CLIPVisionEncode",
|
||||
"_meta": {
|
||||
"title": "CLIP Vision Encode"
|
||||
}
|
||||
},
|
||||
"113": {
|
||||
"inputs": {
|
||||
"clip_name": "sigclip_vision_patch14_384.safetensors"
|
||||
},
|
||||
"class_type": "CLIPVisionLoader",
|
||||
"_meta": {
|
||||
"title": "Load CLIP Vision"
|
||||
}
|
||||
},
|
||||
"114": {
|
||||
"inputs": {
|
||||
"image": "440fb3cb2bcc993bdc7da34986a7135d.jpg"
|
||||
},
|
||||
"class_type": "LoadImage",
|
||||
"_meta": {
|
||||
"title": "Load Image"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -88,3 +88,78 @@ export async function downloadPinterestImages(keyword: string, numberOfPages: nu
|
||||
logger.debug('Done.');
|
||||
return downloadedImagePaths;
|
||||
}
|
||||
|
||||
export async function downloadImagesFromPinterestPin(pinUrl: string, scrollCount: number = 5): Promise<string[]> {
|
||||
const browser = await puppeteer.launch({ headless: false });
|
||||
const page = await browser.newPage();
|
||||
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36');
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
|
||||
const downloadedImagePaths: string[] = [];
|
||||
const downloadedUrls = new Set<string>();
|
||||
|
||||
try {
|
||||
await page.goto(pinUrl, { waitUntil: 'networkidle2' });
|
||||
logger.debug(`Navigated to pin page: ${pinUrl}`);
|
||||
|
||||
const pageImageUrls = new Set<string>();
|
||||
|
||||
for (let i = 0; i < scrollCount; i++) {
|
||||
logger.debug(`Scrolling page ${i + 1}/${scrollCount} for ${pinUrl}...`);
|
||||
const imageUrlsOnPage = await page.evaluate(() => {
|
||||
const images = Array.from(document.querySelectorAll('img[src*="i.pinimg.com"]'));
|
||||
const urls = images.map(img => {
|
||||
const image = img as HTMLImageElement;
|
||||
if (image.srcset) {
|
||||
const sources = image.srcset.split(',').map(s => s.trim().split(' '));
|
||||
const fourXSource = sources.find(s => s[1] === '4x');
|
||||
if (fourXSource) {
|
||||
return fourXSource[0];
|
||||
}
|
||||
}
|
||||
return null; // Return null if no 4x source is found
|
||||
});
|
||||
return urls.filter((url): url is string => url !== null); // Filter out nulls and assert type
|
||||
});
|
||||
|
||||
for (const url of imageUrlsOnPage) {
|
||||
pageImageUrls.add(url);
|
||||
}
|
||||
|
||||
const previousHeight = await page.evaluate('document.body.scrollHeight');
|
||||
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
|
||||
|
||||
try {
|
||||
await page.waitForFunction(`document.body.scrollHeight > ${previousHeight}`, { timeout: 10000 });
|
||||
} catch (e) {
|
||||
logger.debug('No more content to load on this page.');
|
||||
break;
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for images to load
|
||||
}
|
||||
|
||||
let imageCount = 0;
|
||||
for (const imageUrl of pageImageUrls) {
|
||||
if (!downloadedUrls.has(imageUrl)) {
|
||||
downloadedUrls.add(imageUrl);
|
||||
const pinIdMatch = pinUrl.match(/\/pin\/(\d+)\/?/);
|
||||
const basePinId = pinIdMatch ? pinIdMatch[1] : `image_${Date.now()}`;
|
||||
|
||||
const extension = path.extname(new URL(imageUrl).pathname) || '.jpg';
|
||||
const filename = `${basePinId}_related_${imageCount++}${extension}`;
|
||||
const filepath = path.join(downloadPath, filename);
|
||||
|
||||
logger.debug(`Downloading ${imageUrl} to ${filepath}`);
|
||||
await downloadImage(imageUrl, filepath);
|
||||
downloadedImagePaths.push(filepath);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.error(`An error occurred while processing pin ${pinUrl}:`, error);
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
logger.debug('Done processing pin.');
|
||||
return downloadedImagePaths;
|
||||
}
|
||||
|
||||
110
src/lib/openai.ts
Normal file
110
src/lib/openai.ts
Normal file
@ -0,0 +1,110 @@
|
||||
import fs from 'fs';
|
||||
import dotenv from 'dotenv';
|
||||
import { logger } from './logger';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
|
||||
const OPENAI_MODEL = process.env.OPENAI_MODEL;
|
||||
|
||||
async function callOpenAI(prompt: string): Promise<any> {
|
||||
if (!OPENAI_API_KEY || !OPENAI_MODEL) {
|
||||
throw new Error('OPENAI_API_KEY or OPENAI_MODEL is not defined in the .env file');
|
||||
}
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
let llmResponse = "";
|
||||
|
||||
try {
|
||||
const response = await fetch('https://api.openai.com/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${OPENAI_API_KEY}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: OPENAI_MODEL,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
},
|
||||
],
|
||||
temperature: 0.7,
|
||||
}),
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
if (data.choices && data.choices.length > 0) {
|
||||
const content = data.choices[0].message.content;
|
||||
llmResponse = content;
|
||||
const jsonMatch = content.match(/\{[\s\S]*\}/);
|
||||
if (jsonMatch) {
|
||||
return JSON.parse(jsonMatch[0]);
|
||||
}
|
||||
} else {
|
||||
logger.error('Unexpected API response:', data);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Attempt ${i + 1} failed:`, error);
|
||||
logger.debug(`LLM response: ${llmResponse}`)
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Failed to get response from LLM after 10 attempts');
|
||||
}
|
||||
|
||||
async function callOpenAIWithFile(imagePath: string, prompt: string): Promise<any> {
|
||||
if (!OPENAI_API_KEY || !OPENAI_MODEL) {
|
||||
throw new Error('OPENAI_API_KEY or OPENAI_MODEL is not defined in the .env file');
|
||||
}
|
||||
|
||||
const imageBuffer = fs.readFileSync(imagePath);
|
||||
const base64Image = imageBuffer.toString('base64');
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
let llmResponse = "";
|
||||
|
||||
try {
|
||||
const response = await fetch('https://api.openai.com/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${OPENAI_API_KEY}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: OPENAI_MODEL,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'image_url', image_url: { url: `data:image/jpeg;base64,${base64Image}` } },
|
||||
{ type: 'text', text: prompt },
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature: 0.7,
|
||||
}),
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
if (data.choices && data.choices.length > 0) {
|
||||
const content = data.choices[0].message.content;
|
||||
llmResponse = content;
|
||||
const jsonMatch = content.match(/\{[\s\S]*\}/);
|
||||
if (jsonMatch) {
|
||||
return JSON.parse(jsonMatch[0]);
|
||||
}
|
||||
} else {
|
||||
logger.error('Unexpected API response:', data);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Attempt ${i + 1} failed:`, error);
|
||||
logger.debug(`LLM response: ${llmResponse}`)
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Failed to describe image after 10 attempts');
|
||||
}
|
||||
|
||||
export { callOpenAI, callOpenAIWithFile };
|
||||
93
src/lib/video-generator-styled.ts
Normal file
93
src/lib/video-generator-styled.ts
Normal file
@ -0,0 +1,93 @@
|
||||
import * as fs from 'fs/promises';
|
||||
import * as path from 'path';
|
||||
import axios from 'axios';
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
interface VideoSize {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
async function generateStyledVideo(
|
||||
imagePrompt: string,
|
||||
videoPrompt: string,
|
||||
imageName: string,
|
||||
newFileName: string,
|
||||
comfyBaseUrl: string,
|
||||
comfyOutputDir: string,
|
||||
size: VideoSize = { width: 320, height: 640 }
|
||||
): Promise<string> {
|
||||
const COMFY_BASE_URL = comfyBaseUrl.replace(/\/$/, '');
|
||||
const COMFY_OUTPUT_DIR = comfyOutputDir;
|
||||
const workflow = JSON.parse(await fs.readFile('src/comfyworkflows/prototyping_style_flux_wan22.json', 'utf-8'));
|
||||
|
||||
// Set prompts
|
||||
workflow['95']['inputs']['text'] = imagePrompt;
|
||||
workflow['105']['inputs']['text'] = videoPrompt;
|
||||
|
||||
// Set image name
|
||||
workflow['114']['inputs']['image'] = imageName;
|
||||
|
||||
// Set sizes
|
||||
workflow['104']['inputs']['width'] = size.width;
|
||||
workflow['104']['inputs']['height'] = size.height;
|
||||
workflow['97']['inputs']['width'] = size.width;
|
||||
workflow['97']['inputs']['height'] = size.height;
|
||||
|
||||
const response = await axios.post(`${COMFY_BASE_URL}/prompt`, { prompt: workflow });
|
||||
const promptId = response.data.prompt_id;
|
||||
|
||||
let history;
|
||||
do {
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
const historyResponse = await axios.get(`${COMFY_BASE_URL}/history/${promptId}`);
|
||||
history = historyResponse.data[promptId];
|
||||
} while (!history || Object.keys(history.outputs).length === 0);
|
||||
|
||||
const files = await fs.readdir(COMFY_OUTPUT_DIR!);
|
||||
|
||||
// Find the latest MP4 file
|
||||
const generatedVideos = files.filter(file => file.endsWith('.mp4'));
|
||||
const videoStats = await Promise.all(
|
||||
generatedVideos.map(async (file) => {
|
||||
const stat = await fs.stat(path.join(COMFY_OUTPUT_DIR!, file));
|
||||
return { file, mtime: stat.mtime };
|
||||
})
|
||||
);
|
||||
videoStats.sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
|
||||
const latestVideo = videoStats[0].file;
|
||||
|
||||
// Find the latest PNG file
|
||||
const generatedImages = files.filter(file => file.endsWith('.png'));
|
||||
const imageStats = await Promise.all(
|
||||
generatedImages.map(async (file) => {
|
||||
const stat = await fs.stat(path.join(COMFY_OUTPUT_DIR!, file));
|
||||
return { file, mtime: stat.mtime };
|
||||
})
|
||||
);
|
||||
imageStats.sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
|
||||
const latestImage = imageStats[0].file;
|
||||
|
||||
// Create new file paths
|
||||
const newVideoPath = path.resolve('./generated', newFileName);
|
||||
const newImagePath = path.resolve('./generated', newFileName.replace('.mp4', '.png'));
|
||||
|
||||
await fs.mkdir('./generated', { recursive: true });
|
||||
|
||||
// Copy both files
|
||||
const sourceVideoPath = path.join(COMFY_OUTPUT_DIR!, latestVideo);
|
||||
await fs.copyFile(sourceVideoPath, newVideoPath);
|
||||
|
||||
const sourceImagePath = path.join(COMFY_OUTPUT_DIR!, latestImage);
|
||||
await fs.copyFile(sourceImagePath, newImagePath);
|
||||
|
||||
// Optionally, unlink the source files
|
||||
// await fs.unlink(sourceVideoPath);
|
||||
// await fs.unlink(sourceImagePath);
|
||||
|
||||
return newVideoPath;
|
||||
}
|
||||
|
||||
export { generateStyledVideo };
|
||||
272
src/piterest_styletransfer_video.ts
Normal file
272
src/piterest_styletransfer_video.ts
Normal file
@ -0,0 +1,272 @@
|
||||
import { downloadImagesFromPinterestPin } from './lib/downloader';
|
||||
import { callOpenAIWithFile } from './lib/openai';
|
||||
import { generateStyledVideo } from './lib/video-generator-styled';
|
||||
import { logger } from './lib/logger';
|
||||
import * as fs from 'fs/promises';
|
||||
import dotenv from 'dotenv';
|
||||
import path from 'path';
|
||||
import puppeteer from 'puppeteer';
|
||||
import { VideoModel } from './lib/db/video';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const servers = [
|
||||
{
|
||||
baseUrl: process.env.SERVER1_COMFY_BASE_URL,
|
||||
outputDir: process.env.SERVER1_COMFY_OUTPUT_DIR,
|
||||
},
|
||||
/*
|
||||
{
|
||||
baseUrl: process.env.SERVER2_COMFY_BASE_URL,
|
||||
outputDir: process.env.SERVER2_COMFY_OUTPUT_DIR,
|
||||
},
|
||||
*/
|
||||
].filter((s): s is { baseUrl: string; outputDir: string } => !!s.baseUrl && !!s.outputDir);
|
||||
|
||||
interface GenerationTask {
|
||||
pinUrl: string;
|
||||
imagePrompt: string;
|
||||
videoPrompt: string;
|
||||
imageFileName: string;
|
||||
renamedImagePath: string;
|
||||
genre: string;
|
||||
subGenre: string;
|
||||
scene: string;
|
||||
action: string;
|
||||
camera: string;
|
||||
}
|
||||
|
||||
async function getPromptsForImage(imagePath: string, pinUrl: string, genre: string, subGenre: string): Promise<GenerationTask | null> {
|
||||
const pinId = pinUrl.split('/').filter(Boolean).pop() || `pin_${Date.now()}`;
|
||||
const timestamp = new Date().getTime();
|
||||
const imageFileName = `${pinId}_${timestamp}.png`;
|
||||
const renamedImagePath = path.join(path.dirname(imagePath), imageFileName);
|
||||
|
||||
try {
|
||||
await fs.rename(imagePath, renamedImagePath);
|
||||
logger.debug(`Renamed ${imagePath} to ${renamedImagePath}`);
|
||||
|
||||
const promptResponse = await callOpenAIWithFile(renamedImagePath,
|
||||
`Analyze the provided image and generate the following:
|
||||
1. 'scene': A description of the image's environment.
|
||||
2. 'action': A description of the main action occurring in the image.
|
||||
3. 'camera': A description of the camera shot (e.g., 'close-up', 'wide-angle').
|
||||
4. 'image_prompt': A highly detailed, creative, and artistic prompt for an image generation model, inspired by the original. This prompt should be around 200 words.
|
||||
5. 'video_prompt': A prompt for an 8-second video, describing a creative and subtle movement of the main object. The camera should be static, but slight panning is acceptable.
|
||||
|
||||
Output should be in this JSON format:
|
||||
---
|
||||
{
|
||||
"scene": "{result comes here}",
|
||||
"action": "{result comes here}",
|
||||
"camera": "{result comes here}",
|
||||
"image_prompt": "Ultra detailed illustration, {result comes here}",
|
||||
"video_prompt": "{result comes here}"
|
||||
}
|
||||
---
|
||||
`);
|
||||
const { scene, action, camera, image_prompt: imagePrompt, video_prompt: videoPrompt } = promptResponse;
|
||||
logger.info(`Image prompt for ${renamedImagePath}:`, imagePrompt);
|
||||
logger.info(`Video prompt for ${renamedImagePath}:`, videoPrompt);
|
||||
|
||||
return { pinUrl, imagePrompt, videoPrompt, imageFileName, renamedImagePath, genre, subGenre, scene, action, camera };
|
||||
} catch (error) {
|
||||
logger.error(`Failed to get prompts for ${renamedImagePath}:`, error);
|
||||
try {
|
||||
await fs.unlink(renamedImagePath);
|
||||
} catch (cleanupError) {
|
||||
// ignore
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function generateImageAndVideo(task: GenerationTask, server: { baseUrl: string; outputDir: string; }): Promise<{ imagePath: string; videoPath: string; } | null> {
|
||||
const { imagePrompt, videoPrompt, imageFileName, renamedImagePath } = task;
|
||||
const { baseUrl, outputDir } = server;
|
||||
const inputDir = outputDir.replace("output", "input");
|
||||
|
||||
try {
|
||||
const destPath = path.join(inputDir, imageFileName);
|
||||
await fs.copyFile(renamedImagePath, destPath);
|
||||
logger.info(`Copied ${renamedImagePath} to ${destPath}`);
|
||||
|
||||
const videoFileName = imageFileName.replace('.png', '.mp4');
|
||||
await generateStyledVideo(
|
||||
imagePrompt,
|
||||
videoPrompt,
|
||||
imageFileName,
|
||||
videoFileName,
|
||||
baseUrl,
|
||||
outputDir,
|
||||
{ width: 320, height: 640 }
|
||||
);
|
||||
const videoPath = path.join(outputDir, videoFileName);
|
||||
return { imagePath: destPath, videoPath };
|
||||
} catch (error) {
|
||||
logger.error(`Failed to generate styled video for ${imageFileName} on server ${baseUrl}:`, error);
|
||||
return null;
|
||||
} finally {
|
||||
try {
|
||||
await fs.unlink(renamedImagePath);
|
||||
logger.debug(`Deleted renamed source image: ${renamedImagePath}`);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to delete renamed source image ${renamedImagePath}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function worker(id: number, server: { baseUrl: string; outputDir: string; }, taskQueue: GenerationTask[]) {
|
||||
logger.info(`Worker ${id} started for server ${server.baseUrl}`);
|
||||
while (taskQueue.length > 0) {
|
||||
const task = taskQueue.shift();
|
||||
if (task) {
|
||||
logger.info(`Worker ${id} processing task: ${task.imageFileName}`);
|
||||
const result = await generateImageAndVideo(task, server);
|
||||
if (result) {
|
||||
try {
|
||||
const videoData = {
|
||||
genre: task.genre,
|
||||
sub_genre: task.subGenre,
|
||||
scene: task.scene,
|
||||
action: task.action,
|
||||
camera: task.camera,
|
||||
image_prompt: task.imagePrompt,
|
||||
video_prompt: task.videoPrompt,
|
||||
image_path: result.imagePath,
|
||||
video_path: result.videoPath,
|
||||
};
|
||||
const videoId = await VideoModel.create(videoData);
|
||||
logger.info(`Successfully saved video record to database with ID: ${videoId}`);
|
||||
|
||||
const newImageName = `${videoId}_${task.genre}_${task.subGenre}${path.extname(result.imagePath)}`;
|
||||
const newVideoName = `${videoId}_${task.genre}_${task.subGenre}${path.extname(result.videoPath)}`;
|
||||
|
||||
const newImagePath = path.join(path.dirname(result.imagePath), newImageName);
|
||||
const newVideoPath = path.join(path.dirname(result.videoPath), newVideoName);
|
||||
|
||||
await fs.rename(result.imagePath, newImagePath);
|
||||
await fs.rename(result.videoPath, newVideoPath);
|
||||
|
||||
await VideoModel.update(videoId, {
|
||||
image_path: newImagePath,
|
||||
video_path: newVideoPath,
|
||||
});
|
||||
logger.info(`Renamed files and updated database record for video ID: ${videoId}`);
|
||||
} catch (error) {
|
||||
logger.error('Failed to save video record to database or rename files:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.info(`Worker ${id} finished.`);
|
||||
}
|
||||
|
||||
async function getPinUrlFromPinterest(keyword: string): Promise<string | null> {
|
||||
const browser = await puppeteer.launch({ headless: true });
|
||||
const page = await browser.newPage();
|
||||
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36');
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
try {
|
||||
const searchUrl = `https://www.pinterest.com/search/pins/?q=${encodeURIComponent(keyword)}`;
|
||||
await page.goto(searchUrl, { waitUntil: 'networkidle2' });
|
||||
|
||||
const scrollCount = Math.floor(Math.random() * 5) + 1;
|
||||
logger.info(`Scrolling ${scrollCount} times...`);
|
||||
for (let i = 0; i < scrollCount; i++) {
|
||||
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)');
|
||||
await new Promise(resolve => setTimeout(resolve, Math.random() * 1000 + 1000));
|
||||
}
|
||||
|
||||
const pinLinks = await page.$$eval('a', (anchors) =>
|
||||
anchors.map((a) => a.href).filter((href) => href.includes('/pin/'))
|
||||
);
|
||||
|
||||
if (pinLinks.length > 0) {
|
||||
return pinLinks[Math.floor(Math.random() * pinLinks.length)];
|
||||
}
|
||||
return null;
|
||||
} catch (error) {
|
||||
logger.error('Error while getting pin URL from Pinterest:', error);
|
||||
return null;
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const keywords = [
|
||||
{ genre: "fantasy", subGenre: "ethereal" },
|
||||
{ genre: "fantasy", subGenre: "academia" },
|
||||
{ genre: "fantasy", subGenre: "darkacademia" },
|
||||
{ genre: "fantasy", subGenre: "illumication" },
|
||||
{ genre: "fantasy", subGenre: "aesthetic" },
|
||||
{ genre: "abstract", subGenre: "particle" },
|
||||
{ genre: "abstract", subGenre: "space" },
|
||||
{ genre: "abstract", subGenre: "science" },
|
||||
{ genre: "abstract", subGenre: "sphere" },
|
||||
{ genre: "abstract", subGenre: "cubic" },
|
||||
];
|
||||
|
||||
if (servers.length === 0) {
|
||||
logger.error("No servers configured. Please check your .env file.");
|
||||
return;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
|
||||
for (const genreSubGenre of keywords) {
|
||||
|
||||
const { genre, subGenre } = genreSubGenre;
|
||||
|
||||
const keyword = `${genre} ${subGenre}`;
|
||||
logger.info(`Searching for a pin with keyword: ${keyword}`);
|
||||
const pin = await getPinUrlFromPinterest(keyword);
|
||||
|
||||
if (!pin) {
|
||||
logger.warn(`Could not find a pin for keyword: ${keyword}. Skipping.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const generationTasks: GenerationTask[] = [];
|
||||
|
||||
logger.info(`--- Starting processing for pin: ${pin} ---`);
|
||||
const downloadedImagePaths = await downloadImagesFromPinterestPin(pin);
|
||||
if (downloadedImagePaths.length === 0) {
|
||||
logger.warn(`No images were downloaded for pin ${pin}. Skipping.`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const selectedImages = downloadedImagePaths.sort(() => 0.5 - Math.random()).slice(0, 1);
|
||||
logger.info(`--- Randomly selected ${selectedImages.length} images for processing ---`);
|
||||
|
||||
for (const imagePath of selectedImages) {
|
||||
const pinId = path.basename(imagePath, path.extname(imagePath)).split('_related_')[0];
|
||||
const pinUrl = `https://www.pinterest.com/pin/${pinId}/`;
|
||||
const task = await getPromptsForImage(imagePath, pinUrl, genre, subGenre);
|
||||
if (task) {
|
||||
generationTasks.push(task);
|
||||
}
|
||||
}
|
||||
|
||||
const unselectedImages = downloadedImagePaths.filter(p => !selectedImages.includes(p));
|
||||
for (const imagePath of unselectedImages) {
|
||||
try {
|
||||
await fs.unlink(imagePath);
|
||||
logger.debug(`Deleted unselected image: ${imagePath}`);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to delete unselected image ${imagePath}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
if (generationTasks.length > 0) {
|
||||
logger.info(`--- Starting parallel generation of ${generationTasks.length} tasks across ${servers.length} servers ---`);
|
||||
const workers = servers.map((server, index) => worker(index + 1, server, generationTasks));
|
||||
await Promise.all(workers);
|
||||
logger.info("--- Finished parallel generation ---");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
})();
|
||||
Reference in New Issue
Block a user