From b04fc14a6f41fd550e35d7da8979037dcac08c90 Mon Sep 17 00:00:00 2001
From: Ken Yasue <ken@yasue.us>
Date: Fri, 22 Aug 2025 22:42:03 +0200
Subject: [PATCH] update the logic to generate video on t2v

---
 src/generateVideo.ts            |   5 +-
 src/lib/video-generator-text.ts | 104 ++++++++++++++++++++++++++++++++
 2 files changed, 106 insertions(+), 3 deletions(-)
 create mode 100644 src/lib/video-generator-text.ts

diff --git a/src/generateVideo.ts b/src/generateVideo.ts
index c2c4292..b62df22 100644
--- a/src/generateVideo.ts
+++ b/src/generateVideo.ts
@@ -1,5 +1,6 @@
 import { query } from './lib/mysql';
-import { generateVideo } from './lib/video-generator';
+//import { generateVideo } from './lib/video-generator';
+import { generateVideo } from './lib/video-generator-text';
 import { logger } from './lib/logger';
 import dotenv from 'dotenv';
 import path from 'path';
@@ -80,8 +81,6 @@ async function worker(server: any) {
         const videosToProcess = (await query(
             `SELECT * FROM video 
                 WHERE video_prompt IS NOT NULL 
-                    AND image_path IS NOT NULL
-                    AND image_path != 'processing' 
                     AND (video_path IS NULL OR video_path = '') 
                 ORDER BY RAND() LIMIT 1`
         )) as VideoRecord[];
diff --git a/src/lib/video-generator-text.ts b/src/lib/video-generator-text.ts
new file mode 100644
index 0000000..b9af5ad
--- /dev/null
+++ b/src/lib/video-generator-text.ts
@@ -0,0 +1,104 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import axios from 'axios';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+interface VideoSize {
+    width: number;
+    height: number;
+}
+
+async function generateVideo(
+    prompt: string,
+    imagePath: string,
+    newFileName: string,
+    comfyBaseUrl: string,
+    comfyOutputDir: string,
+    size: VideoSize = { width: 720, height: 1280 }
+): Promise<string> {
+    const COMFY_BASE_URL = comfyBaseUrl.replace(/\/$/, '');
+    const COMFY_OUTPUT_DIR = comfyOutputDir;
+    const workflow = JSON.parse(await fs.readFile('src/comfyworkflows/generate_video_text.json', 'utf-8'));
+    workflow['27']['inputs']['text'] = prompt;
+    workflow['28']['inputs']['width'] = size.width;
+    workflow['28']['inputs']['height'] = size.height;
+
+    const response = await axios.post(`${COMFY_BASE_URL}/prompt`, { prompt: workflow });
+    const promptId = response.data.prompt_id;
+
+    let history;
+    do {
+        await new Promise(resolve => setTimeout(resolve, 1000));
+        const historyResponse = await axios.get(`${COMFY_BASE_URL}/history/${promptId}`);
+        history = historyResponse.data[promptId];
+    } while (!history || Object.keys(history.outputs).length === 0);
+
+    const files = await fs.readdir(COMFY_OUTPUT_DIR!);
+
+    // find latest .mp4 file (video) in the comfy output dir
+    const generatedFiles = files.filter(file => file.endsWith('.mp4'));
+
+    const fileStats = await Promise.all(
+        generatedFiles.map(async (file) => {
+            const stat = await fs.stat(path.join(COMFY_OUTPUT_DIR!, file));
+            return { file, mtime: stat.mtime };
+        })
+    );
+
+    if (fileStats.length === 0) {
+        throw new Error('No generated mp4 files found in comfy output directory.');
+    }
+
+    fileStats.sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
+
+    const latestFile = fileStats[0].file;
+    const newFilePath = path.resolve('./generated', newFileName);
+
+    await fs.mkdir('./generated', { recursive: true });
+
+    const sourcePath = path.join(COMFY_OUTPUT_DIR!, latestFile);
+    await fs.copyFile(sourcePath, newFilePath);
+
+    // Handle the static image (.png)
+    // Expected image name: same base name as the video, but .png extension
+    const expectedImageName = path.basename(newFileName, path.extname(newFileName)) + '.png';
+
+    const pngFiles = files.filter(file => file.endsWith('.png'));
+
+    let sourcePngFile: string | null = null;
+
+    // Prefer exact match (Comfy sometimes names the image exactly as the video base name)
+    if (pngFiles.includes(expectedImageName)) {
+        sourcePngFile = expectedImageName;
+    } else if (pngFiles.length > 0) {
+        // Fallback: pick the most recent .png by timestamp
+        const pngStats = await Promise.all(
+            pngFiles.map(async (file) => {
+                const stat = await fs.stat(path.join(COMFY_OUTPUT_DIR!, file));
+                return { file, mtime: stat.mtime };
+            })
+        );
+        pngStats.sort((a, b) => b.mtime.getTime() - a.mtime.getTime());
+        sourcePngFile = pngStats[0].file;
+    }
+
+    if (sourcePngFile) {
+        const targetPngPath = path.resolve('./generated', expectedImageName);
+        // Delete existing target image if present
+        try {
+            await fs.unlink(targetPngPath);
+        } catch (err) {
+            // ignore if not exists
+        }
+
+        const sourcePngPath = path.join(COMFY_OUTPUT_DIR!, sourcePngFile);
+        // Copy and rename the png to the generated folder with the expected name
+        await fs.copyFile(sourcePngPath, targetPngPath);
+    }
+
+    return newFilePath;
+}
+
+export { generateVideo };