save current changes
This commit is contained in:
81
src/generateImage.ts
Normal file
81
src/generateImage.ts
Normal file
@ -0,0 +1,81 @@
|
||||
import { query } from './lib/mysql';
|
||||
import { generateImage } from './lib/image-generator';
|
||||
import { logger } from './lib/logger';
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
interface VideoRecord {
|
||||
id: number;
|
||||
genre: string;
|
||||
sub_genre: string;
|
||||
image_prompt: string;
|
||||
// Add other fields if necessary
|
||||
}
|
||||
|
||||
const servers = [
|
||||
{
|
||||
baseUrl: process.env.SERVER1_COMFY_BASE_URL,
|
||||
outputDir: process.env.SERVER1_COMFY_OUTPUT_DIR,
|
||||
},
|
||||
{
|
||||
baseUrl: process.env.SERVER2_COMFY_BASE_URL,
|
||||
outputDir: process.env.SERVER2_COMFY_OUTPUT_DIR,
|
||||
},
|
||||
];
|
||||
|
||||
async function processServerQueue(server: any, videos: any[]) {
|
||||
for (const video of videos) {
|
||||
try {
|
||||
const fileName = `${video.id}_${video.genre}_${video.sub_genre}.png`.replace(/\s/g, '_');
|
||||
const imagePath = await generateImage(
|
||||
video.image_prompt,
|
||||
fileName,
|
||||
server.baseUrl,
|
||||
server.outputDir,
|
||||
'flux',
|
||||
{ width: 720, height: 1280 }
|
||||
);
|
||||
|
||||
await query('UPDATE video SET image_path = ? WHERE id = ?', [imagePath, video.id]);
|
||||
logger.info(`Generated and saved image for video ${video.id} at ${imagePath}`);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to generate image for video ${video.id}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
const videosToProcess = await query(
|
||||
"SELECT * FROM video WHERE image_prompt IS NOT NULL AND (image_path IS NULL OR image_path = '')"
|
||||
) as any[];
|
||||
|
||||
if (videosToProcess.length === 0) {
|
||||
logger.info('No images to generate.');
|
||||
return;
|
||||
}
|
||||
|
||||
const queues: VideoRecord[][] = servers.map(() => []);
|
||||
videosToProcess.forEach((video, index) => {
|
||||
queues[index % servers.length].push(video);
|
||||
});
|
||||
|
||||
const promises = servers.map((server, index) => {
|
||||
if (!server.baseUrl || !server.outputDir) {
|
||||
logger.warn(`Server ${index + 1} is not configured. Skipping.`);
|
||||
return Promise.resolve();
|
||||
}
|
||||
return processServerQueue(server, queues[index]);
|
||||
});
|
||||
|
||||
await Promise.all(promises);
|
||||
logger.info('Finished generating all images.');
|
||||
} catch (error) {
|
||||
logger.error('An error occurred during image generation:', error);
|
||||
} finally {
|
||||
process.exit();
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
116
src/generatePrompt.ts
Normal file
116
src/generatePrompt.ts
Normal file
@ -0,0 +1,116 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { query } from './lib/mysql';
|
||||
import { logger } from './lib/logger';
|
||||
import { callLMStudio } from './lib/lmstudio';
|
||||
|
||||
async function main() {
|
||||
//await insertGenres();
|
||||
await generatePrompts();
|
||||
process.exit();
|
||||
}
|
||||
|
||||
async function insertGenres() {
|
||||
logger.info('Starting genre insertion...');
|
||||
try {
|
||||
const genresPath = path.join(__dirname, 'data', 'genres.json');
|
||||
const genresData = JSON.parse(fs.readFileSync(genresPath, 'utf-8'));
|
||||
|
||||
for (const genre in genresData) {
|
||||
if (genresData.hasOwnProperty(genre)) {
|
||||
const items = genresData[genre];
|
||||
for (let item of items) {
|
||||
if (Array.isArray(item)) {
|
||||
for (const subItem of item) {
|
||||
await processItem(genre, subItem);
|
||||
}
|
||||
} else {
|
||||
await processItem(genre, item);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
logger.info('Finished processing genres.');
|
||||
} catch (error) {
|
||||
logger.error('Error processing genres:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function generatePrompts() {
|
||||
logger.info('Starting prompt generation...');
|
||||
try {
|
||||
const videosToUpdate = await query(
|
||||
'SELECT * FROM video WHERE image_prompt IS NULL OR video_prompt IS NULL'
|
||||
) as any[];
|
||||
|
||||
for (const video of videosToUpdate) {
|
||||
const prompt = `
|
||||
Based on the following video details, generate an image prompt and a video prompt.
|
||||
Each prompt should be around 100 words.
|
||||
The output should be a JSON object in the format: { "imagePrompt": "...", "videoPrompt": "..." }
|
||||
For video prompt, it takes 8 seconds. So start with current scene, describe action and camera work.
|
||||
Please include lighting details too.
|
||||
|
||||
Details:
|
||||
- Genre: ${video.genre}
|
||||
- Sub-Genre: ${video.sub_genre}
|
||||
- Scene: ${video.scene}
|
||||
- Action: ${video.action}
|
||||
- Camera: ${video.camera}
|
||||
`;
|
||||
|
||||
logger.info(`Generating prompt for video ID: ${video.id}`);
|
||||
logger.debug(prompt);
|
||||
|
||||
let success = false;
|
||||
for (let i = 0; i < 10; i++) {
|
||||
try {
|
||||
const response = await callLMStudio(prompt);
|
||||
if (response && response.imagePrompt && response.videoPrompt) {
|
||||
await query(
|
||||
'UPDATE video SET image_prompt = ?, video_prompt = ? WHERE id = ?',
|
||||
[response.imagePrompt, response.videoPrompt, video.id]
|
||||
);
|
||||
logger.info(`Successfully updated prompts for video ID: ${video.id}`);
|
||||
success = true;
|
||||
break;
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn(`Attempt ${i + 1} failed for video ID ${video.id}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
logger.error(`Failed to generate and parse prompts for video ID: ${video.id} after 10 attempts.`);
|
||||
}
|
||||
}
|
||||
logger.info('Finished generating prompts.');
|
||||
} catch (error) {
|
||||
logger.error('Error generating prompts:', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function processItem(genre: string, item: any) {
|
||||
if (item && typeof item === 'object' && !Array.isArray(item)) {
|
||||
const { subGenre, scene, action, camera } = item;
|
||||
|
||||
// Check if the item already exists
|
||||
const existing = await query(
|
||||
'SELECT id FROM video WHERE genre = ? AND sub_genre = ? AND scene = ? AND action = ? AND camera = ?',
|
||||
[genre, subGenre, scene, action, camera]
|
||||
) as any[];
|
||||
|
||||
if (existing.length === 0) {
|
||||
// Insert the new item
|
||||
await query(
|
||||
'INSERT INTO video (genre, sub_genre, scene, action, camera) VALUES (?, ?, ?, ?, ?)',
|
||||
[genre, subGenre, scene, action, camera]
|
||||
);
|
||||
logger.info(`Inserted: ${genre} - ${subGenre} - ${scene}`);
|
||||
} else {
|
||||
logger.info(`Skipped (exists): ${genre} - ${subGenre} - ${scene}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@ -3,6 +3,12 @@ import { generateVideo } from './lib/video-generator';
|
||||
import { logger } from './lib/logger';
|
||||
import { Scene } from './types';
|
||||
import scenes from './scenes/space.json';
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const COMFY_BASE_URL = process.env.COMFY_BASE_URL;
|
||||
const COMFY_OUTPUT_DIR = process.env.COMFY_OUTPUT_DIR;
|
||||
|
||||
interface ProcessedScene {
|
||||
scene: Scene;
|
||||
@ -10,6 +16,11 @@ interface ProcessedScene {
|
||||
}
|
||||
|
||||
async function prepareImageForScene(scene: Scene): Promise<ProcessedScene | null> {
|
||||
if (!COMFY_BASE_URL || !COMFY_OUTPUT_DIR) {
|
||||
logger.error('COMFY_BASE_URL and COMFY_OUTPUT_DIR must be set in the environment variables.');
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const startTime = Date.now();
|
||||
logger.info(`--- Preparing image for scene: ${scene.idea} ---`);
|
||||
@ -19,6 +30,8 @@ async function prepareImageForScene(scene: Scene): Promise<ProcessedScene | null
|
||||
const generatedImagePath = await generateImage(
|
||||
scene.image_prompt,
|
||||
imageFileName,
|
||||
COMFY_BASE_URL,
|
||||
COMFY_OUTPUT_DIR,
|
||||
'flux',
|
||||
{ width: 1280, height: 720 }
|
||||
);
|
||||
@ -35,6 +48,11 @@ async function prepareImageForScene(scene: Scene): Promise<ProcessedScene | null
|
||||
}
|
||||
|
||||
async function generateVideoForScene(processedScene: ProcessedScene) {
|
||||
if (!COMFY_BASE_URL || !COMFY_OUTPUT_DIR) {
|
||||
logger.error('COMFY_BASE_URL and COMFY_OUTPUT_DIR must be set in the environment variables.');
|
||||
return;
|
||||
}
|
||||
|
||||
const { scene, generatedImagePath } = processedScene;
|
||||
try {
|
||||
const startTime = Date.now();
|
||||
@ -45,6 +63,8 @@ async function generateVideoForScene(processedScene: ProcessedScene) {
|
||||
scene.video_prompt,
|
||||
generatedImagePath,
|
||||
videoFileName,
|
||||
COMFY_BASE_URL,
|
||||
COMFY_OUTPUT_DIR,
|
||||
{ width: 1280, height: 720 }
|
||||
);
|
||||
const endTime = Date.now();
|
||||
|
||||
53
src/lib/db/schema.ts
Normal file
53
src/lib/db/schema.ts
Normal file
@ -0,0 +1,53 @@
|
||||
import { query } from '../mysql';
|
||||
|
||||
async function createSchema() {
|
||||
try {
|
||||
console.log('Creating tables...');
|
||||
|
||||
await query(`
|
||||
CREATE TABLE IF NOT EXISTS video (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
genre VARCHAR(255),
|
||||
sub_genre VARCHAR(255),
|
||||
scene TEXT,
|
||||
action TEXT,
|
||||
camera TEXT,
|
||||
image_prompt TEXT,
|
||||
video_prompt TEXT,
|
||||
image_path VARCHAR(255),
|
||||
video_path VARCHAR(255)
|
||||
)
|
||||
`);
|
||||
|
||||
await query(`
|
||||
CREATE TABLE IF NOT EXISTS tag (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
tag VARCHAR(255) UNIQUE
|
||||
)
|
||||
`);
|
||||
|
||||
await query(`
|
||||
CREATE TABLE IF NOT EXISTS tag_video (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
tag_id INT,
|
||||
video_id INT,
|
||||
FOREIGN KEY (tag_id) REFERENCES tag(id),
|
||||
FOREIGN KEY (video_id) REFERENCES video(id)
|
||||
)
|
||||
`);
|
||||
|
||||
console.log('Tables created successfully.');
|
||||
} catch (error) {
|
||||
console.error('Error creating tables:', error);
|
||||
} finally {
|
||||
process.exit();
|
||||
}
|
||||
}
|
||||
|
||||
createSchema();
|
||||
58
src/lib/db/tag.ts
Normal file
58
src/lib/db/tag.ts
Normal file
@ -0,0 +1,58 @@
|
||||
import { query } from '../mysql';
|
||||
|
||||
export interface Tag {
|
||||
id: number;
|
||||
created_at: Date;
|
||||
modified_at: Date;
|
||||
tag: string;
|
||||
}
|
||||
|
||||
export class TagModel {
|
||||
static async create(tag: Omit<Tag, 'id' | 'created_at' | 'modified_at'>): Promise<number> {
|
||||
const sql = 'INSERT INTO tag (tag) VALUES (?)';
|
||||
const params = [tag.tag];
|
||||
const result: any = await query(sql, params);
|
||||
return result.insertId;
|
||||
}
|
||||
|
||||
static async getById(id: number): Promise<Tag | null> {
|
||||
const sql = 'SELECT * FROM tag WHERE id = ?';
|
||||
const params = [id];
|
||||
const rows: any = await query(sql, params);
|
||||
return rows.length > 0 ? rows[0] as Tag : null;
|
||||
}
|
||||
|
||||
static async getByTag(tag: string): Promise<Tag | null> {
|
||||
const sql = 'SELECT * FROM tag WHERE tag = ?';
|
||||
const params = [tag];
|
||||
const rows: any = await query(sql, params);
|
||||
return rows.length > 0 ? rows[0] as Tag : null;
|
||||
}
|
||||
|
||||
static async getAll(): Promise<Tag[]> {
|
||||
const sql = 'SELECT * FROM tag';
|
||||
const rows: any = await query(sql);
|
||||
return rows as Tag[];
|
||||
}
|
||||
|
||||
static async update(id: number, tag: Partial<Omit<Tag, 'id' | 'created_at' | 'modified_at'>>): Promise<boolean> {
|
||||
const fields = Object.keys(tag);
|
||||
const values = Object.values(tag);
|
||||
|
||||
if (fields.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const sql = `UPDATE tag SET ${fields.map(field => `${field} = ?`).join(', ')} WHERE id = ?`;
|
||||
const params = [...values, id];
|
||||
const result: any = await query(sql, params);
|
||||
return result.affectedRows > 0;
|
||||
}
|
||||
|
||||
static async delete(id: number): Promise<boolean> {
|
||||
const sql = 'DELETE FROM tag WHERE id = ?';
|
||||
const params = [id];
|
||||
const result: any = await query(sql, params);
|
||||
return result.affectedRows > 0;
|
||||
}
|
||||
}
|
||||
46
src/lib/db/tag_video.ts
Normal file
46
src/lib/db/tag_video.ts
Normal file
@ -0,0 +1,46 @@
|
||||
import { query } from '../mysql';
|
||||
|
||||
export interface TagVideo {
|
||||
id: number;
|
||||
created_at: Date;
|
||||
modified_at: Date;
|
||||
tag_id: number;
|
||||
video_id: number;
|
||||
}
|
||||
|
||||
export class TagVideoModel {
|
||||
static async create(tagVideo: Omit<TagVideo, 'id' | 'created_at' | 'modified_at'>): Promise<number> {
|
||||
const sql = 'INSERT INTO tag_video (tag_id, video_id) VALUES (?, ?)';
|
||||
const params = [tagVideo.tag_id, tagVideo.video_id];
|
||||
const result: any = await query(sql, params);
|
||||
return result.insertId;
|
||||
}
|
||||
|
||||
static async getByVideoId(videoId: number): Promise<TagVideo[]> {
|
||||
const sql = 'SELECT * FROM tag_video WHERE video_id = ?';
|
||||
const params = [videoId];
|
||||
const rows: any = await query(sql, params);
|
||||
return rows as TagVideo[];
|
||||
}
|
||||
|
||||
static async getByTagId(tagId: number): Promise<TagVideo[]> {
|
||||
const sql = 'SELECT * FROM tag_video WHERE tag_id = ?';
|
||||
const params = [tagId];
|
||||
const rows: any = await query(sql, params);
|
||||
return rows as TagVideo[];
|
||||
}
|
||||
|
||||
static async delete(id: number): Promise<boolean> {
|
||||
const sql = 'DELETE FROM tag_video WHERE id = ?';
|
||||
const params = [id];
|
||||
const result: any = await query(sql, params);
|
||||
return result.affectedRows > 0;
|
||||
}
|
||||
|
||||
static async deleteByTagAndVideo(tagId: number, videoId: number): Promise<boolean> {
|
||||
const sql = 'DELETE FROM tag_video WHERE tag_id = ? AND video_id = ?';
|
||||
const params = [tagId, videoId];
|
||||
const result: any = await query(sql, params);
|
||||
return result.affectedRows > 0;
|
||||
}
|
||||
}
|
||||
59
src/lib/db/video.ts
Normal file
59
src/lib/db/video.ts
Normal file
@ -0,0 +1,59 @@
|
||||
import { query } from '../mysql';
|
||||
|
||||
export interface Video {
|
||||
id: number;
|
||||
created_at: Date;
|
||||
modified_at: Date;
|
||||
genre: string;
|
||||
sub_genre: string;
|
||||
scene: string;
|
||||
action: string;
|
||||
camera: string;
|
||||
image_prompt: string;
|
||||
video_prompt: string;
|
||||
image_path: string;
|
||||
video_path: string;
|
||||
}
|
||||
|
||||
export class VideoModel {
|
||||
static async create(video: Omit<Video, 'id' | 'created_at' | 'modified_at'>): Promise<number> {
|
||||
const sql = 'INSERT INTO video (genre, sub_genre, scene, action, camera, image_prompt, video_prompt, image_path, video_path) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)';
|
||||
const params = [video.genre, video.sub_genre, video.scene, video.action, video.camera, video.image_prompt, video.video_prompt, video.image_path, video.video_path];
|
||||
const result: any = await query(sql, params);
|
||||
return result.insertId;
|
||||
}
|
||||
|
||||
static async getById(id: number): Promise<Video | null> {
|
||||
const sql = 'SELECT * FROM video WHERE id = ?';
|
||||
const params = [id];
|
||||
const rows: any = await query(sql, params);
|
||||
return rows.length > 0 ? rows[0] as Video : null;
|
||||
}
|
||||
|
||||
static async getAll(): Promise<Video[]> {
|
||||
const sql = 'SELECT * FROM video';
|
||||
const rows: any = await query(sql);
|
||||
return rows as Video[];
|
||||
}
|
||||
|
||||
static async update(id: number, video: Partial<Omit<Video, 'id' | 'created_at' | 'modified_at'>>): Promise<boolean> {
|
||||
const fields = Object.keys(video);
|
||||
const values = Object.values(video);
|
||||
|
||||
if (fields.length === 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const sql = `UPDATE video SET ${fields.map(field => `${field} = ?`).join(', ')} WHERE id = ?`;
|
||||
const params = [...values, id];
|
||||
const result: any = await query(sql, params);
|
||||
return result.affectedRows > 0;
|
||||
}
|
||||
|
||||
static async delete(id: number): Promise<boolean> {
|
||||
const sql = 'DELETE FROM video WHERE id = ?';
|
||||
const params = [id];
|
||||
const result: any = await query(sql, params);
|
||||
return result.affectedRows > 0;
|
||||
}
|
||||
}
|
||||
@ -1,66 +1,7 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import dotenv from 'dotenv';
|
||||
import { logger } from './logger';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const LLM_BASE_URL = process.env.LLM_BASE_URL;
|
||||
import { callLMStudioWithFile } from './lmstudio';
|
||||
|
||||
async function describeImage(imagePath: string, prompt: string): Promise<any> {
|
||||
if (!LLM_BASE_URL) {
|
||||
throw new Error('LLM_BASE_URL is not defined in the .env file');
|
||||
}
|
||||
|
||||
const imageBuffer = fs.readFileSync(imagePath);
|
||||
const base64Image = imageBuffer.toString('base64');
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
let llmResponse = "";
|
||||
|
||||
try {
|
||||
const requestUrl = new URL('v1/chat/completions', LLM_BASE_URL);
|
||||
const response = await fetch(requestUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'local-model',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'image_url', image_url: { url: `data:image/jpeg;base64,${base64Image}` } },
|
||||
{ type: 'text', text: prompt },
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature: 0.7,
|
||||
}),
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
if (data.choices && data.choices.length > 0) {
|
||||
const content = data.choices[0].message.content;
|
||||
llmResponse = content;
|
||||
const jsonMatch = content.match(/\{[\s\S]*\}/);
|
||||
if (jsonMatch) {
|
||||
return JSON.parse(jsonMatch[0]);
|
||||
}
|
||||
} else {
|
||||
logger.error('Unexpected API response:', data);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Attempt ${i + 1} failed:`, error);
|
||||
if (error instanceof TypeError && error.message.includes('fetch failed')) {
|
||||
logger.error('Could not connect to the LM Studio server. Please ensure the server is running and accessible at the specified LLM_BASE_URL.');
|
||||
}
|
||||
logger.debug(`LLM response: ${llmResponse}`)
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Failed to describe image after 10 attempts');
|
||||
return await callLMStudioWithFile(imagePath, prompt);
|
||||
}
|
||||
|
||||
export { describeImage };
|
||||
|
||||
@ -5,9 +5,6 @@ import dotenv from 'dotenv';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const COMFY_BASE_URL = process.env.COMFY_BASE_URL?.replace(/\/$/, '');
|
||||
const COMFY_OUTPUT_DIR = process.env.COMFY_OUTPUT_DIR;
|
||||
|
||||
interface ImageSize {
|
||||
width: number;
|
||||
height: number;
|
||||
@ -16,9 +13,13 @@ interface ImageSize {
|
||||
async function generateImage(
|
||||
prompt: string,
|
||||
newFileName: string,
|
||||
comfyBaseUrl: string,
|
||||
comfyOutputDir: string,
|
||||
imageModel: 'qwen' | 'flux' = 'qwen',
|
||||
size: ImageSize = { width: 720, height: 1280 }
|
||||
): Promise<string> {
|
||||
const COMFY_BASE_URL = comfyBaseUrl.replace(/\/$/, '');
|
||||
const COMFY_OUTPUT_DIR = comfyOutputDir;
|
||||
let workflow;
|
||||
if (imageModel === 'qwen') {
|
||||
workflow = JSON.parse(await fs.readFile('src/comfyworkflows/generate_image.json', 'utf-8'));
|
||||
|
||||
115
src/lib/lmstudio.ts
Normal file
115
src/lib/lmstudio.ts
Normal file
@ -0,0 +1,115 @@
|
||||
import fs from 'fs';
|
||||
import dotenv from 'dotenv';
|
||||
import { logger } from './logger';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const LLM_BASE_URL = process.env.LLM_BASE_URL;
|
||||
|
||||
async function callLMStudio(prompt: string): Promise<any> {
|
||||
if (!LLM_BASE_URL) {
|
||||
throw new Error('LLM_BASE_URL is not defined in the .env file');
|
||||
}
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
let llmResponse = "";
|
||||
|
||||
try {
|
||||
const requestUrl = new URL('v1/chat/completions', LLM_BASE_URL);
|
||||
const response = await fetch(requestUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'local-model',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
},
|
||||
],
|
||||
temperature: 0.7,
|
||||
}),
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
if (data.choices && data.choices.length > 0) {
|
||||
const content = data.choices[0].message.content;
|
||||
llmResponse = content;
|
||||
const jsonMatch = content.match(/\{[\s\S]*\}/);
|
||||
if (jsonMatch) {
|
||||
return JSON.parse(jsonMatch[0]);
|
||||
}
|
||||
} else {
|
||||
logger.error('Unexpected API response:', data);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Attempt ${i + 1} failed:`, error);
|
||||
if (error instanceof TypeError && error.message.includes('fetch failed')) {
|
||||
logger.error('Could not connect to the LM Studio server. Please ensure the server is running and accessible at the specified LLM_BASE_URL.');
|
||||
}
|
||||
logger.debug(`LLM response: ${llmResponse}`)
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Failed to get response from LLM after 10 attempts');
|
||||
}
|
||||
|
||||
async function callLMStudioWithFile(imagePath: string, prompt: string): Promise<any> {
|
||||
if (!LLM_BASE_URL) {
|
||||
throw new Error('LLM_BASE_URL is not defined in the .env file');
|
||||
}
|
||||
|
||||
const imageBuffer = fs.readFileSync(imagePath);
|
||||
const base64Image = imageBuffer.toString('base64');
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
let llmResponse = "";
|
||||
|
||||
try {
|
||||
const requestUrl = new URL('v1/chat/completions', LLM_BASE_URL);
|
||||
const response = await fetch(requestUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: 'local-model',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'image_url', image_url: { url: `data:image/jpeg;base64,${base64Image}` } },
|
||||
{ type: 'text', text: prompt },
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature: 0.7,
|
||||
}),
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
if (data.choices && data.choices.length > 0) {
|
||||
const content = data.choices[0].message.content;
|
||||
llmResponse = content;
|
||||
const jsonMatch = content.match(/\{[\s\S]*\}/);
|
||||
if (jsonMatch) {
|
||||
return JSON.parse(jsonMatch[0]);
|
||||
}
|
||||
} else {
|
||||
logger.error('Unexpected API response:', data);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Attempt ${i + 1} failed:`, error);
|
||||
if (error instanceof TypeError && error.message.includes('fetch failed')) {
|
||||
logger.error('Could not connect to the LM Studio server. Please ensure the server is running and accessible at the specified LLM_BASE_URL.');
|
||||
}
|
||||
logger.debug(`LLM response: ${llmResponse}`)
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Failed to describe image after 10 attempts');
|
||||
}
|
||||
|
||||
export { callLMStudio, callLMStudioWithFile };
|
||||
33
src/lib/mysql.ts
Normal file
33
src/lib/mysql.ts
Normal file
@ -0,0 +1,33 @@
|
||||
import mysql from 'mysql2/promise';
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const pool = mysql.createPool({
|
||||
host: process.env.MYSQL_HOST,
|
||||
user: process.env.MYSQL_USER,
|
||||
password: process.env.MYSQL_PASSWOWD,
|
||||
database: process.env.MYSQL_DB,
|
||||
waitForConnections: true,
|
||||
connectionLimit: 10,
|
||||
queueLimit: 0
|
||||
});
|
||||
|
||||
export async function query(sql: string, params: any[] = []) {
|
||||
const [rows] = await pool.execute(sql, params);
|
||||
return rows;
|
||||
}
|
||||
|
||||
export async function testConnection() {
|
||||
try {
|
||||
const connection = await pool.getConnection();
|
||||
console.log('Successfully connected to the database.');
|
||||
connection.release();
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error('Failed to connect to the database:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export default pool;
|
||||
@ -5,9 +5,6 @@ import dotenv from 'dotenv';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const COMFY_BASE_URL = process.env.COMFY_BASE_URL?.replace(/\/$/, '');
|
||||
const COMFY_OUTPUT_DIR = process.env.COMFY_OUTPUT_DIR;
|
||||
|
||||
interface VideoSize {
|
||||
width: number;
|
||||
height: number;
|
||||
@ -17,8 +14,12 @@ async function generateVideo(
|
||||
prompt: string,
|
||||
imagePath: string,
|
||||
newFileName: string,
|
||||
comfyBaseUrl: string,
|
||||
comfyOutputDir: string,
|
||||
size: VideoSize = { width: 720, height: 1280 }
|
||||
): Promise<string> {
|
||||
const COMFY_BASE_URL = comfyBaseUrl.replace(/\/$/, '');
|
||||
const COMFY_OUTPUT_DIR = comfyOutputDir;
|
||||
const workflow = JSON.parse(await fs.readFile('src/comfyworkflows/generate_video.json', 'utf-8'));
|
||||
workflow['6']['inputs']['text'] = prompt;
|
||||
workflow['52']['inputs']['image'] = imagePath;
|
||||
|
||||
69
src/testmysql.ts
Normal file
69
src/testmysql.ts
Normal file
@ -0,0 +1,69 @@
|
||||
import { VideoModel } from './lib/db/video';
|
||||
import { TagModel } from './lib/db/tag';
|
||||
import { TagVideoModel } from './lib/db/tag_video';
|
||||
import pool from './lib/mysql';
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
console.log('Starting CRUD operations test...');
|
||||
|
||||
// Create a video
|
||||
const videoData = {
|
||||
genre: 'Sci-Fi',
|
||||
sub_genre: 'Cyberpunk',
|
||||
scene: 'A rainy night in a futuristic city',
|
||||
action: 'A detective chasing a rogue android',
|
||||
camera: 'Low-angle shot, neon lights reflecting on wet pavement',
|
||||
image_prompt: 'cyberpunk city rain neon',
|
||||
video_prompt: 'detective chase android rain',
|
||||
image_path: '/images/test.png',
|
||||
video_path: '/videos/test.mp4',
|
||||
};
|
||||
const videoId = await VideoModel.create(videoData);
|
||||
console.log(`Created video with ID: ${videoId}`);
|
||||
|
||||
// Create a tag
|
||||
const tagData = { tag: 'cyberpunk' };
|
||||
const tagId = await TagModel.create(tagData);
|
||||
console.log(`Created tag with ID: ${tagId}`);
|
||||
|
||||
// Associate tag with video
|
||||
const tagVideoId = await TagVideoModel.create({ tag_id: tagId, video_id: videoId });
|
||||
console.log(`Associated tag ${tagId} with video ${videoId}. Association ID: ${tagVideoId}`);
|
||||
|
||||
// Retrieve and verify
|
||||
const video = await VideoModel.getById(videoId);
|
||||
console.log('Retrieved video:', video);
|
||||
const tag = await TagModel.getById(tagId);
|
||||
console.log('Retrieved tag:', tag);
|
||||
const videoTags = await TagVideoModel.getByVideoId(videoId);
|
||||
console.log('Retrieved video tags:', videoTags);
|
||||
|
||||
// Update
|
||||
await VideoModel.update(videoId, { scene: 'A sunny day in a futuristic city' });
|
||||
const updatedVideo = await VideoModel.getById(videoId);
|
||||
console.log('Updated video:', updatedVideo);
|
||||
|
||||
await TagModel.update(tagId, { tag: 'dystopian' });
|
||||
const updatedTag = await TagModel.getById(tagId);
|
||||
console.log('Updated tag:', updatedTag);
|
||||
|
||||
// Cleanup
|
||||
console.log('Cleaning up created records...');
|
||||
await TagVideoModel.delete(tagVideoId);
|
||||
console.log(`Deleted tag_video association with ID: ${tagVideoId}`);
|
||||
await VideoModel.delete(videoId);
|
||||
console.log(`Deleted video with ID: ${videoId}`);
|
||||
await TagModel.delete(tagId);
|
||||
console.log(`Deleted tag with ID: ${tagId}`);
|
||||
|
||||
console.log('Test completed successfully.');
|
||||
} catch (error) {
|
||||
console.error('An error occurred during the test:', error);
|
||||
} finally {
|
||||
await pool.end();
|
||||
process.exit();
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user