anosatsuk124/VoiceRSSSummary
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply formatting

Browse Source
This commit is contained in:
Satsuki Akiba
2025-06-08 15:21:58 +09:00
parent b5ff912fcb
commit a728ebb66c
28 changed files with 1809 additions and 1137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

203
scripts/fetch_and_generate.ts
View File

@@ -39,7 +39,7 @@ export async function batchProcess(abortSignal?: AbortSignal): Promise<void> {
// Check for cancellation at start
if (abortSignal?.aborted) {
throw new Error('Batch process was cancelled before starting');
throw new Error("Batch process was cancelled before starting");
}
// Load feed URLs from file
@@ -55,14 +55,17 @@ export async function batchProcess(abortSignal?: AbortSignal): Promise<void> {
for (const url of feedUrls) {
// Check for cancellation before processing each feed
if (abortSignal?.aborted) {
throw new Error('Batch process was cancelled during feed processing');
throw new Error("Batch process was cancelled during feed processing");
}
try {
await processFeedUrl(url, abortSignal);
} catch (error) {
// Re-throw cancellation errors
if (error instanceof Error && (error.message.includes('cancelled') || error.name === 'AbortError')) {
if (
error instanceof Error &&
(error.message.includes("cancelled") || error.name === "AbortError")
) {
throw error;
}
console.error(`❌ Failed to process feed ${url}:`, error);
@@ -72,7 +75,7 @@ export async function batchProcess(abortSignal?: AbortSignal): Promise<void> {
// Check for cancellation before processing articles
if (abortSignal?.aborted) {
throw new Error('Batch process was cancelled before article processing');
throw new Error("Batch process was cancelled before article processing");
}
// Process unprocessed articles and generate podcasts
@@ -83,10 +86,13 @@ export async function batchProcess(abortSignal?: AbortSignal): Promise<void> {
new Date().toISOString(),
);
} catch (error) {
if (error instanceof Error && (error.message.includes('cancelled') || error.name === 'AbortError')) {
if (
error instanceof Error &&
(error.message.includes("cancelled") || error.name === "AbortError")
) {
console.log("🛑 Batch process was cancelled");
const abortError = new Error('Batch process was cancelled');
abortError.name = 'AbortError';
const abortError = new Error("Batch process was cancelled");
abortError.name = "AbortError";
throw abortError;
}
console.error("💥 Batch process failed:", error);
@@ -116,14 +122,17 @@ async function loadFeedUrls(): Promise<string[]> {
/**
* Process a single feed URL and discover new articles
*/
async function processFeedUrl(url: string, abortSignal?: AbortSignal): Promise<void> {
async function processFeedUrl(
url: string,
abortSignal?: AbortSignal,
): Promise<void> {
if (!url || !url.startsWith("http")) {
throw new Error(`Invalid feed URL: ${url}`);
}
// Check for cancellation
if (abortSignal?.aborted) {
throw new Error('Feed processing was cancelled');
throw new Error("Feed processing was cancelled");
}
console.log(`🔍 Processing feed: ${url}`);
@@ -132,10 +141,10 @@ async function processFeedUrl(url: string, abortSignal?: AbortSignal): Promise<v
// Parse RSS feed
const parser = new Parser<FeedItem>();
const feed = await parser.parseURL(url);
// Check for cancellation after parsing
if (abortSignal?.aborted) {
throw new Error('Feed processing was cancelled');
throw new Error("Feed processing was cancelled");
}
// Get or create feed record
@@ -225,13 +234,15 @@ async function discoverNewArticles(
/**
* Process unprocessed articles and generate podcasts
*/
async function processUnprocessedArticles(abortSignal?: AbortSignal): Promise<void> {
async function processUnprocessedArticles(
abortSignal?: AbortSignal,
): Promise<void> {
console.log("🎧 Processing unprocessed articles...");
try {
// Check for cancellation
if (abortSignal?.aborted) {
throw new Error('Article processing was cancelled');
throw new Error("Article processing was cancelled");
}
// Process retry queue first
@@ -239,7 +250,7 @@ async function processUnprocessedArticles(abortSignal?: AbortSignal): Promise<vo
// Check for cancellation after retry queue
if (abortSignal?.aborted) {
throw new Error('Article processing was cancelled');
throw new Error("Article processing was cancelled");
}
// Get unprocessed articles (limit to prevent overwhelming)
@@ -260,31 +271,44 @@ async function processUnprocessedArticles(abortSignal?: AbortSignal): Promise<vo
for (const article of unprocessedArticles) {
// Check for cancellation before processing each article
if (abortSignal?.aborted) {
throw new Error('Article processing was cancelled');
throw new Error("Article processing was cancelled");
}
try {
const episodeCreated = await generatePodcastForArticle(article, abortSignal);
const episodeCreated = await generatePodcastForArticle(
article,
abortSignal,
);
// Only mark as processed and update RSS if episode was actually created
if (episodeCreated) {
await markArticleAsProcessed(article.id);
console.log(`✅ Podcast generated for: ${article.title}`);
successfullyGeneratedCount++;
// Update RSS immediately after each successful episode creation
console.log(`📻 Updating podcast RSS after successful episode creation...`);
console.log(
`📻 Updating podcast RSS after successful episode creation...`,
);
try {
await updatePodcastRSS();
console.log(`📻 RSS updated successfully for: ${article.title}`);
} catch (rssError) {
console.error(`❌ Failed to update RSS after episode creation for: ${article.title}`, rssError);
console.error(
`❌ Failed to update RSS after episode creation for: ${article.title}`,
rssError,
);
}
} else {
console.warn(`⚠️ Episode creation failed for: ${article.title} - not marking as processed`);
console.warn(
`⚠️ Episode creation failed for: ${article.title} - not marking as processed`,
);
}
} catch (error) {
if (error instanceof Error && (error.message.includes('cancelled') || error.name === 'AbortError')) {
if (
error instanceof Error &&
(error.message.includes("cancelled") || error.name === "AbortError")
) {
console.log(`🛑 Article processing cancelled, stopping batch`);
throw error; // Re-throw to propagate cancellation
}
@@ -296,7 +320,9 @@ async function processUnprocessedArticles(abortSignal?: AbortSignal): Promise<vo
}
}
console.log(`🎯 Batch processing completed: ${successfullyGeneratedCount} episodes successfully created`);
console.log(
`🎯 Batch processing completed: ${successfullyGeneratedCount} episodes successfully created`,
);
if (successfullyGeneratedCount === 0) {
console.log(`ℹ️ No episodes were successfully created in this batch`);
}
@@ -310,15 +336,16 @@ async function processUnprocessedArticles(abortSignal?: AbortSignal): Promise<vo
* Process retry queue for failed TTS generation
*/
async function processRetryQueue(abortSignal?: AbortSignal): Promise<void> {
const { getQueueItems, updateQueueItemStatus, removeFromQueue } = await import("../services/database.js");
const { getQueueItems, updateQueueItemStatus, removeFromQueue } =
await import("../services/database.js");
const { Database } = await import("bun:sqlite");
const db = new Database(config.paths.dbPath);
console.log("🔄 Processing TTS retry queue...");
try {
const queueItems = await getQueueItems(5); // Process 5 items at a time
if (queueItems.length === 0) {
return;
}
@@ -328,53 +355,75 @@ async function processRetryQueue(abortSignal?: AbortSignal): Promise<void> {
for (const item of queueItems) {
// Check for cancellation before processing each retry item
if (abortSignal?.aborted) {
throw new Error('Retry queue processing was cancelled');
throw new Error("Retry queue processing was cancelled");
}
try {
console.log(`🔁 Retrying TTS generation for: ${item.itemId} (attempt ${item.retryCount + 1}/3)`);
console.log(
`🔁 Retrying TTS generation for: ${item.itemId} (attempt ${item.retryCount + 1}/3)`,
);
// Mark as processing
await updateQueueItemStatus(item.id, 'processing');
await updateQueueItemStatus(item.id, "processing");
// Attempt TTS generation without re-queuing on failure
const audioFilePath = await generateTTSWithoutQueue(item.itemId, item.scriptText, item.retryCount);
const audioFilePath = await generateTTSWithoutQueue(
item.itemId,
item.scriptText,
item.retryCount,
);
// Success - remove from queue and update RSS
await removeFromQueue(item.id);
console.log(`✅ TTS retry successful for: ${item.itemId}`);
// Update RSS immediately after successful retry
console.log(`📻 Updating podcast RSS after successful retry...`);
try {
await updatePodcastRSS();
console.log(`📻 RSS updated successfully after retry for: ${item.itemId}`);
console.log(
`📻 RSS updated successfully after retry for: ${item.itemId}`,
);
} catch (rssError) {
console.error(`❌ Failed to update RSS after retry for: ${item.itemId}`, rssError);
console.error(
`❌ Failed to update RSS after retry for: ${item.itemId}`,
rssError,
);
}
} catch (error) {
if (error instanceof Error && (error.message.includes('cancelled') || error.name === 'AbortError')) {
if (
error instanceof Error &&
(error.message.includes("cancelled") || error.name === "AbortError")
) {
console.log(`🛑 TTS retry processing cancelled for: ${item.itemId}`);
throw error; // Re-throw cancellation errors
}
console.error(`❌ TTS retry failed for: ${item.itemId}`, error);
try {
if (item.retryCount >= 2) {
// Max retries reached, mark as failed
await updateQueueItemStatus(item.id, 'failed');
console.log(`💀 Max retries reached for: ${item.itemId}, marking as failed`);
await updateQueueItemStatus(item.id, "failed");
console.log(
`💀 Max retries reached for: ${item.itemId}, marking as failed`,
);
} else {
// Increment retry count and reset to pending for next retry
const updatedRetryCount = item.retryCount + 1;
const stmt = db.prepare("UPDATE tts_queue SET retry_count = ?, status = 'pending' WHERE id = ?");
const stmt = db.prepare(
"UPDATE tts_queue SET retry_count = ?, status = 'pending' WHERE id = ?",
);
stmt.run(updatedRetryCount, item.id);
console.log(`🔄 Updated retry count to ${updatedRetryCount} for: ${item.itemId}`);
console.log(
`🔄 Updated retry count to ${updatedRetryCount} for: ${item.itemId}`,
);
}
} catch (dbError) {
console.error(`❌ Failed to update queue status for: ${item.itemId}`, dbError);
console.error(
`❌ Failed to update queue status for: ${item.itemId}`,
dbError,
);
}
}
}
@@ -386,7 +435,10 @@ async function processRetryQueue(abortSignal?: AbortSignal): Promise<void> {
try {
db.close();
} catch (closeError) {
console.warn("⚠️ Warning: Failed to close database connection:", closeError);
console.warn(
"⚠️ Warning: Failed to close database connection:",
closeError,
);
}
}
}
@@ -395,13 +447,16 @@ async function processRetryQueue(abortSignal?: AbortSignal): Promise<void> {
* Generate podcast for a single article
* Returns true if episode was successfully created, false otherwise
*/
async function generatePodcastForArticle(article: any, abortSignal?: AbortSignal): Promise<boolean> {
async function generatePodcastForArticle(
article: any,
abortSignal?: AbortSignal,
): Promise<boolean> {
console.log(`🎤 Generating podcast for: ${article.title}`);
try {
// Check for cancellation
if (abortSignal?.aborted) {
throw new Error('Podcast generation was cancelled');
throw new Error("Podcast generation was cancelled");
}
// Get feed information for context
@@ -410,7 +465,7 @@ async function generatePodcastForArticle(article: any, abortSignal?: AbortSignal
// Check for cancellation before classification
if (abortSignal?.aborted) {
throw new Error('Podcast generation was cancelled');
throw new Error("Podcast generation was cancelled");
}
// Classify the article/feed
@@ -421,7 +476,7 @@ async function generatePodcastForArticle(article: any, abortSignal?: AbortSignal
// Check for cancellation before content generation
if (abortSignal?.aborted) {
throw new Error('Podcast generation was cancelled');
throw new Error("Podcast generation was cancelled");
}
// Enhance article content with web scraping if needed
@@ -430,7 +485,7 @@ async function generatePodcastForArticle(article: any, abortSignal?: AbortSignal
article.title,
article.link,
article.content,
article.description
article.description,
);
// Generate podcast content for this single article
@@ -442,10 +497,10 @@ async function generatePodcastForArticle(article: any, abortSignal?: AbortSignal
description: enhancedContent.description,
},
]);
// Check for cancellation before TTS
if (abortSignal?.aborted) {
throw new Error('Podcast generation was cancelled');
throw new Error("Podcast generation was cancelled");
}
// Generate unique ID for the episode
@@ -458,15 +513,20 @@ async function generatePodcastForArticle(article: any, abortSignal?: AbortSignal
console.log(`🔊 Audio generated: ${audioFilePath}`);
} catch (ttsError) {
console.error(`❌ TTS generation failed for ${article.title}:`, ttsError);
// Check if error indicates item was added to retry queue
const errorMessage = ttsError instanceof Error ? ttsError.message : String(ttsError);
if (errorMessage.includes('added to retry queue')) {
console.log(`📋 Article will be retried later via TTS queue: ${article.title}`);
const errorMessage =
ttsError instanceof Error ? ttsError.message : String(ttsError);
if (errorMessage.includes("added to retry queue")) {
console.log(
`📋 Article will be retried later via TTS queue: ${article.title}`,
);
// Don't mark as processed - leave it for retry
return false;
} else {
console.error(`💀 TTS generation permanently failed for ${article.title} - max retries exceeded`);
console.error(
`💀 TTS generation permanently failed for ${article.title} - max retries exceeded`,
);
// Max retries exceeded, don't create episode but mark as processed to avoid infinite retry
return false;
}
@@ -476,11 +536,16 @@ async function generatePodcastForArticle(article: any, abortSignal?: AbortSignal
try {
const audioStats = await getAudioFileStats(audioFilePath);
if (audioStats.size === 0) {
console.error(`❌ Audio file is empty for ${article.title}: ${audioFilePath}`);
console.error(
`❌ Audio file is empty for ${article.title}: ${audioFilePath}`,
);
return false;
}
} catch (statsError) {
console.error(`❌ Cannot access audio file for ${article.title}: ${audioFilePath}`, statsError);
console.error(
`❌ Cannot access audio file for ${article.title}: ${audioFilePath}`,
statsError,
);
return false;
}
@@ -502,11 +567,17 @@ async function generatePodcastForArticle(article: any, abortSignal?: AbortSignal
console.log(`💾 Episode saved for article: ${article.title}`);
return true;
} catch (saveError) {
console.error(`❌ Failed to save episode for ${article.title}:`, saveError);
console.error(
`❌ Failed to save episode for ${article.title}:`,
saveError,
);
return false;
}
} catch (error) {
if (error instanceof Error && (error.message.includes('cancelled') || error.name === 'AbortError')) {
if (
error instanceof Error &&
(error.message.includes("cancelled") || error.name === "AbortError")
) {
console.log(`🛑 Podcast generation cancelled for: ${article.title}`);
throw error; // Re-throw cancellation errors to stop the batch
}