1059 lines
37 KiB
TypeScript
1059 lines
37 KiB
TypeScript
import puppeteer, { type Browser } from "puppeteer";
|
|
import * as cheerio from "cheerio";
|
|
import type { CheerioAPI } from "cheerio";
|
|
|
|
export interface ExtractedContent {
|
|
title?: string;
|
|
content: string;
|
|
description?: string;
|
|
success: boolean;
|
|
error?: string;
|
|
}
|
|
|
|
interface RetryOptions {
|
|
maxRetries: number;
|
|
baseDelay: number;
|
|
maxDelay: number;
|
|
backoffMultiplier: number;
|
|
}
|
|
|
|
const DEFAULT_RETRY_OPTIONS: RetryOptions = {
|
|
maxRetries: 3,
|
|
baseDelay: 1000,
|
|
maxDelay: 10000,
|
|
backoffMultiplier: 2
|
|
};
|
|
|
|
// Singleton browser instance for reuse
|
|
let sharedBrowser: Browser | null = null;
|
|
|
|
// Helper function to replace page.waitForTimeout
|
|
async function waitForTimeout(ms: number): Promise<void> {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
// Dynamic content handling function
|
|
async function handleDynamicContent(page: any): Promise<void> {
|
|
try {
|
|
console.log('Starting dynamic content handling...');
|
|
// Wait for initial content
|
|
await page.waitForSelector('body', { timeout: 5000 });
|
|
|
|
// Progressive loading strategy
|
|
const loadingStrategies = [
|
|
// Strategy 1: Wait for common loading indicators to disappear
|
|
async () => {
|
|
const loadingSelectors = [
|
|
'.loading', '.loader', '.spinner', '.skeleton',
|
|
'[class*="loading"]', '[class*="skeleton"]',
|
|
'.placeholder', '.shimmer'
|
|
];
|
|
|
|
for (const selector of loadingSelectors) {
|
|
try {
|
|
await page.waitForSelector(selector, { timeout: 2000 });
|
|
await page.waitForSelector(selector, { hidden: true, timeout: 10000 });
|
|
break;
|
|
} catch (e) {
|
|
// Continue to next selector
|
|
}
|
|
}
|
|
},
|
|
|
|
// Strategy 2: Auto-scroll to trigger lazy loading
|
|
async () => {
|
|
await page.evaluate(() => {
|
|
return new Promise<void>((resolve) => {
|
|
let totalHeight = 0;
|
|
const distance = 500;
|
|
const timer = setInterval(() => {
|
|
const scrollHeight = document.body.scrollHeight;
|
|
window.scrollBy(0, distance);
|
|
totalHeight += distance;
|
|
|
|
if (totalHeight >= scrollHeight || totalHeight > 5000) {
|
|
clearInterval(timer);
|
|
window.scrollTo(0, 0); // Scroll back to top
|
|
setTimeout(() => resolve(), 1000);
|
|
}
|
|
}, 200);
|
|
});
|
|
});
|
|
},
|
|
|
|
// Strategy 3: Wait for content-specific indicators
|
|
async () => {
|
|
const contentSelectors = [
|
|
'article', '.article-content', '.post-content', '.entry-content',
|
|
'main', '[role="main"]', '.main-content'
|
|
];
|
|
|
|
for (const selector of contentSelectors) {
|
|
try {
|
|
await page.waitForSelector(selector, { timeout: 3000 });
|
|
break;
|
|
} catch (e) {
|
|
// Continue to next selector
|
|
}
|
|
}
|
|
},
|
|
|
|
// Strategy 4: Handle "Read More" or expansion buttons
|
|
async () => {
|
|
const expandButtons = [
|
|
'button[class*="read-more"]', 'button[class*="expand"]',
|
|
'.read-more', '.show-more', '.expand-content',
|
|
'a[class*="read-more"]', 'a[class*="continue"]'
|
|
];
|
|
|
|
for (const selector of expandButtons) {
|
|
try {
|
|
const button = await page.$(selector);
|
|
if (button) {
|
|
await button.click();
|
|
await waitForTimeout(2000);
|
|
break;
|
|
}
|
|
} catch (e) {
|
|
// Continue to next button
|
|
}
|
|
}
|
|
}
|
|
];
|
|
|
|
// Execute strategies with timeouts
|
|
const executeWithTimeout = async (strategy: () => Promise<void>, timeout: number) => {
|
|
return Promise.race([
|
|
strategy(),
|
|
new Promise<void>((resolve) => setTimeout(resolve, timeout))
|
|
]);
|
|
};
|
|
|
|
// Execute all strategies in parallel with timeouts
|
|
await Promise.allSettled([
|
|
executeWithTimeout(loadingStrategies[0]!, 3000),
|
|
executeWithTimeout(loadingStrategies[1]!, 8000),
|
|
executeWithTimeout(loadingStrategies[2]!, 5000),
|
|
executeWithTimeout(loadingStrategies[3]!, 3000)
|
|
]);
|
|
|
|
// Final wait for any remaining dynamic content
|
|
await waitForTimeout(2000);
|
|
|
|
} catch (error) {
|
|
console.log('Dynamic content handling failed, using basic timeout:', error);
|
|
// If dynamic content handling fails, continue with basic timeout
|
|
await waitForTimeout(3000);
|
|
}
|
|
console.log('Dynamic content handling completed.');
|
|
}
|
|
|
|
async function getBrowser(): Promise<Browser> {
|
|
if (!sharedBrowser || !sharedBrowser.isConnected()) {
|
|
sharedBrowser = await puppeteer.launch({
|
|
headless: true,
|
|
args: [
|
|
"--no-sandbox",
|
|
"--disable-setuid-sandbox",
|
|
"--disable-dev-shm-usage",
|
|
"--disable-accelerated-2d-canvas",
|
|
"--no-first-run",
|
|
"--no-zygote",
|
|
"--disable-gpu",
|
|
"--disable-web-security",
|
|
"--disable-features=VizDisplayCompositor",
|
|
"--disable-background-timer-throttling",
|
|
"--disable-backgrounding-occluded-windows",
|
|
"--disable-renderer-backgrounding",
|
|
"--disable-field-trial-config",
|
|
"--disable-ipc-flooding-protection",
|
|
"--enable-automation",
|
|
"--force-device-scale-factor=1",
|
|
"--ignore-certificate-errors",
|
|
"--ignore-ssl-errors",
|
|
"--ignore-certificate-errors-spki-list",
|
|
"--allow-running-insecure-content",
|
|
"--disable-extensions",
|
|
"--no-default-browser-check",
|
|
"--disable-default-apps",
|
|
"--disable-sync",
|
|
"--metrics-recording-only",
|
|
"--no-pings",
|
|
"--mute-audio"
|
|
],
|
|
});
|
|
}
|
|
return sharedBrowser;
|
|
}
|
|
|
|
// Helper function for exponential backoff retry
|
|
async function retryWithBackoff<T>(
|
|
operation: () => Promise<T>,
|
|
options: RetryOptions = DEFAULT_RETRY_OPTIONS,
|
|
attempt: number = 1
|
|
): Promise<T> {
|
|
try {
|
|
return await operation();
|
|
} catch (error) {
|
|
if (attempt >= options.maxRetries) {
|
|
throw error;
|
|
}
|
|
|
|
const isRetryableError = error instanceof Error && (
|
|
error.message.includes('ERR_SOCKET_NOT_CONNECTED') ||
|
|
error.message.includes('ERR_CONNECTION_REFUSED') ||
|
|
error.message.includes('ERR_CONNECTION_RESET') ||
|
|
error.message.includes('ERR_NETWORK_CHANGED') ||
|
|
error.message.includes('ERR_INTERNET_DISCONNECTED') ||
|
|
error.message.includes('ERR_NAME_NOT_RESOLVED') ||
|
|
error.message.includes('ERR_TIMED_OUT') ||
|
|
error.message.includes('Protocol error') ||
|
|
error.message.includes('Navigation timeout') ||
|
|
error.message.includes('net::') ||
|
|
error.message.includes('Target closed') ||
|
|
error.message.includes('Session closed')
|
|
);
|
|
|
|
if (!isRetryableError) {
|
|
throw error;
|
|
}
|
|
|
|
const delay = Math.min(
|
|
options.baseDelay * Math.pow(options.backoffMultiplier, attempt - 1),
|
|
options.maxDelay
|
|
);
|
|
|
|
console.log(`Attempt ${attempt} failed, retrying in ${delay}ms:`, error.message);
|
|
await waitForTimeout(delay);
|
|
|
|
return retryWithBackoff(operation, options, attempt + 1);
|
|
}
|
|
}
|
|
|
|
export async function closeBrowser(): Promise<void> {
|
|
if (sharedBrowser && sharedBrowser.isConnected()) {
|
|
await sharedBrowser.close();
|
|
sharedBrowser = null;
|
|
}
|
|
}
|
|
|
|
// Fallback content extraction using fetch + cheerio
|
|
async function extractWithFetchFallback(url: string): Promise<ExtractedContent> {
|
|
console.log(`Using fetch fallback for: ${url}`);
|
|
|
|
try {
|
|
const userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
|
|
|
|
const response = await fetch(url, {
|
|
headers: {
|
|
'User-Agent': userAgent,
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
'DNT': '1',
|
|
'Connection': 'keep-alive',
|
|
'Upgrade-Insecure-Requests': '1',
|
|
'Cache-Control': 'no-cache'
|
|
},
|
|
signal: AbortSignal.timeout(30000) // 30 second timeout
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
}
|
|
|
|
const html = await response.text();
|
|
const $ = cheerio.load(html);
|
|
|
|
// Remove unwanted elements first
|
|
const unwantedSelectors = [
|
|
"script", "style", "noscript", "iframe", "embed", "object",
|
|
"nav", "header", "footer", "aside", "form",
|
|
".advertisement", ".ads", ".ad", ".adsbygoogle", "[class*='ad-']", "[id*='ad-']",
|
|
".sidebar", ".menu", ".navigation", ".nav", ".breadcrumb",
|
|
".social-share", ".share", ".social", ".sns",
|
|
".comments", ".comment", ".disqus",
|
|
".cookie-banner", ".cookie", ".gdpr",
|
|
".popup", ".modal", ".overlay", ".lightbox",
|
|
".related", ".recommended", ".more-stories",
|
|
".tags", ".categories", ".metadata",
|
|
".author-bio", ".author-info",
|
|
".newsletter", ".subscribe", ".signup",
|
|
"[role='complementary']", "[role='banner']", "[role='contentinfo']",
|
|
"[aria-label*='advertisement']", "[aria-label*='sidebar']"
|
|
];
|
|
|
|
unwantedSelectors.forEach((selector) => {
|
|
$(selector).remove();
|
|
});
|
|
|
|
// Extract title
|
|
let title = "";
|
|
const titleSources = [
|
|
$('meta[property="og:title"]').attr('content'),
|
|
$('meta[name="twitter:title"]').attr('content'),
|
|
$('h1').first().text().trim(),
|
|
$('.article-title, .post-title, .entry-title').first().text().trim(),
|
|
$('title').text().trim(),
|
|
$('[itemprop="headline"]').first().text().trim()
|
|
];
|
|
|
|
for (const titleSource of titleSources) {
|
|
if (titleSource && titleSource.length > 0) {
|
|
title = titleSource;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Extract description
|
|
let description = "";
|
|
const descriptionSources = [
|
|
$('meta[property="og:description"]').attr('content'),
|
|
$('meta[name="description"]').attr('content'),
|
|
$('meta[name="twitter:description"]').attr('content'),
|
|
$('[itemprop="description"]').first().text().trim()
|
|
];
|
|
|
|
for (const descSource of descriptionSources) {
|
|
if (descSource && descSource.length > 0) {
|
|
description = descSource;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Content selectors (same as in Puppeteer version)
|
|
const contentSelectors = [
|
|
'[itemtype*="Article"] [itemprop="articleBody"]',
|
|
'[itemtype*="NewsArticle"] [itemprop="articleBody"]',
|
|
'[itemtype*="BlogPosting"] [itemprop="articleBody"]',
|
|
'article[role="main"]',
|
|
'main article',
|
|
'[role="main"] article',
|
|
'article',
|
|
'.post-content', '.entry-content', '.article-content', '.content-area',
|
|
'.article-body', '.post-body', '.entry-body', '.story-body',
|
|
'.main-content', '.primary-content', '.page-content',
|
|
'.news-content', '.blog-content', '.editorial-content',
|
|
'.wp-content', '.entry', '.post',
|
|
'.section-content', '.postArticle-content', '.post-full-content',
|
|
'.markup', '.section--body', '.section-divider + .section-content',
|
|
'.honbun', '.main_text', '.article_body', '.news_body',
|
|
'.entry_text', '.blog_text', '.content_text',
|
|
'.kiji', '.news', '.article',
|
|
'main', '[role="main"]',
|
|
'#content', '#main', '#article', '#post', '#entry',
|
|
'#main-content', '#primary', '#content-area',
|
|
'.content', '.main', '.wrapper', '.container'
|
|
];
|
|
|
|
// Function to calculate content quality score
|
|
const calculateContentScore = (element: cheerio.Cheerio<any>): number => {
|
|
const text = element.text() || '';
|
|
if (text.length < 100) return 0;
|
|
|
|
let score = 0;
|
|
|
|
// Base score from text length (diminishing returns)
|
|
score += Math.min(text.length / 100, 50);
|
|
|
|
// Paragraph density
|
|
const paragraphs = element.find('p');
|
|
const avgParagraphLength = paragraphs.length > 0 ?
|
|
paragraphs.toArray().reduce((sum, p) => sum + ($(p).text().length || 0), 0) / paragraphs.length : 0;
|
|
|
|
if (avgParagraphLength > 100) score += 20;
|
|
if (paragraphs.length > 3) score += 10;
|
|
|
|
// Link density penalty
|
|
const links = element.find('a');
|
|
const linkText = links.toArray().reduce((sum, link) => sum + ($(link).text().length || 0), 0);
|
|
const linkDensity = text.length > 0 ? linkText / text.length : 0;
|
|
if (linkDensity < 0.2) score += 15;
|
|
else if (linkDensity < 0.4) score += 5;
|
|
else score -= 10;
|
|
|
|
// Bonus for article-like structure
|
|
if (element.prop('tagName') === 'ARTICLE') score += 25;
|
|
if (element.attr('role') === 'main') score += 20;
|
|
if (element.find('h1, h2, h3').length > 0) score += 10;
|
|
|
|
// Bonus for semantic elements
|
|
const semanticElements = element.find('p, h1, h2, h3, h4, h5, h6, blockquote, ul, ol');
|
|
if (semanticElements.length > 5) score += 15;
|
|
|
|
// Penalty for navigation-like content
|
|
const navWords = ['メニュー', 'ナビ', 'カテゴリ', 'タグ', 'menu', 'navigation', 'nav', 'sidebar'];
|
|
const className = (element.attr('class') || '').toLowerCase();
|
|
const id = (element.attr('id') || '').toLowerCase();
|
|
if (navWords.some(word => className.includes(word) || id.includes(word))) {
|
|
score -= 20;
|
|
}
|
|
|
|
return Math.max(score, 0);
|
|
};
|
|
|
|
// Function to clean and normalize text
|
|
const cleanText = (text: string): string => {
|
|
return text
|
|
.replace(/\s+/g, ' ')
|
|
.replace(/\n\s*\n\s*\n/g, '\n\n')
|
|
.replace(/^\s+|\s+$/g, '')
|
|
.replace(/[\u200B-\u200D\uFEFF]/g, '')
|
|
.trim();
|
|
};
|
|
|
|
// Collect and score all content candidates
|
|
interface ContentCandidate {
|
|
element: cheerio.Cheerio<any>;
|
|
score: number;
|
|
content: string;
|
|
selector: string;
|
|
}
|
|
|
|
const candidates: ContentCandidate[] = [];
|
|
|
|
for (const selector of contentSelectors) {
|
|
try {
|
|
const elements = $(selector);
|
|
elements.each((index, element) => {
|
|
const $element = $(element);
|
|
const text = $element.text() || '';
|
|
if (text.length > 200) {
|
|
const score = calculateContentScore($element);
|
|
candidates.push({
|
|
element: $element,
|
|
score,
|
|
content: cleanText(text),
|
|
selector: `${selector}[${index}]`
|
|
});
|
|
}
|
|
});
|
|
} catch (e) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Sort candidates by score (highest first)
|
|
candidates.sort((a, b) => b.score - a.score);
|
|
|
|
console.log(`Found ${candidates.length} content candidates`);
|
|
if (candidates.length > 0) {
|
|
console.log(`Best candidate score: ${candidates[0]!.score}, selector: ${candidates[0]!.selector}`);
|
|
}
|
|
|
|
// Get the best content
|
|
let content = "";
|
|
if (candidates.length > 0) {
|
|
content = candidates[0]!.content;
|
|
|
|
// If the best candidate is still short, try combining top candidates
|
|
if (content.length < 500 && candidates.length > 1) {
|
|
const topCandidates = candidates.slice(0, 3).filter(c => c.score > 10);
|
|
const combinedContent = topCandidates.map(c => c.content).join('\n\n');
|
|
if (combinedContent.length > content.length) {
|
|
content = cleanText(combinedContent);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Fallback strategies if still no good content
|
|
if (!content || content.length < 200) {
|
|
console.log('Using paragraph aggregation fallback...');
|
|
const paragraphs = $('p').toArray()
|
|
.map(p => $(p).text().trim())
|
|
.filter(p => p.length > 50)
|
|
.filter(p => {
|
|
const lowerP = p.toLowerCase();
|
|
return !lowerP.includes('cookie') &&
|
|
!lowerP.includes('privacy') &&
|
|
!lowerP.includes('terms of service') &&
|
|
!lowerP.includes('subscribe') &&
|
|
!lowerP.includes('newsletter');
|
|
});
|
|
|
|
if (paragraphs.length > 0) {
|
|
content = cleanText(paragraphs.join('\n\n'));
|
|
}
|
|
}
|
|
|
|
// Final fallback: structured data
|
|
if (!content || content.length < 200) {
|
|
console.log('Trying structured data fallback...');
|
|
try {
|
|
const jsonLd = $('script[type="application/ld+json"]').first().html();
|
|
if (jsonLd) {
|
|
const data = JSON.parse(jsonLd);
|
|
if (data.articleBody) {
|
|
content = cleanText(data.articleBody);
|
|
} else if (data.text) {
|
|
content = cleanText(data.text);
|
|
}
|
|
}
|
|
} catch (e) {
|
|
// Ignore JSON parsing errors
|
|
}
|
|
}
|
|
|
|
// Limit content length to avoid token limits
|
|
const maxLength = 50000;
|
|
if (content.length > maxLength) {
|
|
content = content.substring(0, maxLength) + "...";
|
|
}
|
|
|
|
console.log(`Fetch fallback extracted content: ${content.length} characters`);
|
|
|
|
if (!content || content.length < 100) {
|
|
return {
|
|
title: title || '',
|
|
content: '',
|
|
description: description || '',
|
|
success: false,
|
|
error: `Insufficient content extracted via fetch fallback (${content?.length || 0} characters)`,
|
|
};
|
|
}
|
|
|
|
return {
|
|
title: title || '',
|
|
content,
|
|
description: description || '',
|
|
success: true,
|
|
};
|
|
|
|
} catch (error) {
|
|
console.error(`Fetch fallback failed:`, error);
|
|
return {
|
|
title: '',
|
|
content: '',
|
|
description: '',
|
|
success: false,
|
|
error: error instanceof Error ? error.message : 'Unknown error in fetch fallback',
|
|
};
|
|
}
|
|
}
|
|
|
|
async function extractWithRetry(url: string): Promise<ExtractedContent> {
|
|
const userAgents = [
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0"
|
|
];
|
|
|
|
return retryWithBackoff(async () => {
|
|
let page = null;
|
|
try {
|
|
const browser = await getBrowser();
|
|
page = await browser.newPage();
|
|
|
|
// Randomize user agent to avoid detection
|
|
const userAgent = userAgents[Math.floor(Math.random() * userAgents.length)] || userAgents[0];
|
|
await page.setUserAgent(userAgent!);
|
|
await page.setViewport({ width: 1280, height: 720 });
|
|
|
|
// Set longer timeout for problematic sites
|
|
page.setDefaultNavigationTimeout(60000);
|
|
page.setDefaultTimeout(60000);
|
|
|
|
// Block unnecessary resources to speed up loading
|
|
await page.setRequestInterception(true);
|
|
page.on('request', (req) => {
|
|
const resourceType = req.resourceType();
|
|
if (resourceType === 'image' || resourceType === 'media' || resourceType === 'font') {
|
|
req.abort();
|
|
} else {
|
|
req.continue();
|
|
}
|
|
});
|
|
|
|
// Add extra headers to appear more like a real browser
|
|
await page.setExtraHTTPHeaders({
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
'DNT': '1',
|
|
'Connection': 'keep-alive',
|
|
'Upgrade-Insecure-Requests': '1'
|
|
});
|
|
|
|
// Navigate with multiple wait strategies
|
|
let response;
|
|
try {
|
|
response = await page.goto(url, {
|
|
waitUntil: ["domcontentloaded", "networkidle0"],
|
|
timeout: 60000,
|
|
});
|
|
} catch (networkError) {
|
|
// Fallback to more basic wait strategy
|
|
const errorMessage = networkError instanceof Error ? networkError.message : 'Unknown error';
|
|
console.log('Network idle failed, trying domcontentloaded only:', errorMessage);
|
|
response = await page.goto(url, {
|
|
waitUntil: "domcontentloaded",
|
|
timeout: 60000,
|
|
});
|
|
}
|
|
|
|
if (!response) {
|
|
throw new Error('No response received from server');
|
|
}
|
|
|
|
const status = response?.status();
|
|
if (status && status >= 400) {
|
|
throw new Error(`HTTP ${status}: ${response?.statusText() || 'Unknown error'}`);
|
|
}
|
|
|
|
// Enhanced dynamic content handling
|
|
console.log('Handling dynamic content...');
|
|
await handleDynamicContent(page);
|
|
|
|
// Extract content using advanced multi-strategy approach
|
|
console.log('Extracting content using multi-strategy approach...');
|
|
const extractedData = await page.evaluate(() => {
|
|
interface ContentCandidate {
|
|
element: Element;
|
|
score: number;
|
|
content: string;
|
|
selector: string;
|
|
}
|
|
|
|
// Remove unwanted elements first
|
|
const unwantedSelectors = [
|
|
"script", "style", "noscript", "iframe", "embed", "object",
|
|
"nav", "header", "footer", "aside", "form",
|
|
".advertisement", ".ads", ".ad", ".adsbygoogle", "[class*='ad-']", "[id*='ad-']",
|
|
".sidebar", ".menu", ".navigation", ".nav", ".breadcrumb",
|
|
".social-share", ".share", ".social", ".sns",
|
|
".comments", ".comment", ".disqus",
|
|
".cookie-banner", ".cookie", ".gdpr",
|
|
".popup", ".modal", ".overlay", ".lightbox",
|
|
".related", ".recommended", ".more-stories",
|
|
".tags", ".categories", ".metadata",
|
|
".author-bio", ".author-info",
|
|
".newsletter", ".subscribe", ".signup",
|
|
"[role='complementary']", "[role='banner']", "[role='contentinfo']",
|
|
"[aria-label*='advertisement']", "[aria-label*='sidebar']"
|
|
];
|
|
|
|
unwantedSelectors.forEach((selector) => {
|
|
try {
|
|
const elements = document.querySelectorAll(selector);
|
|
elements.forEach((el) => el.remove());
|
|
} catch (e) {
|
|
// Ignore invalid selectors
|
|
}
|
|
});
|
|
|
|
// Extract title
|
|
let title = "";
|
|
const titleSources = [
|
|
() => document.querySelector('meta[property="og:title"]')?.getAttribute('content'),
|
|
() => document.querySelector('meta[name="twitter:title"]')?.getAttribute('content'),
|
|
() => document.querySelector('h1')?.textContent?.trim(),
|
|
() => document.querySelector('.article-title, .post-title, .entry-title')?.textContent?.trim(),
|
|
() => document.querySelector('title')?.textContent?.trim(),
|
|
() => document.querySelector('[itemprop="headline"]')?.textContent?.trim()
|
|
];
|
|
|
|
for (const source of titleSources) {
|
|
try {
|
|
const result = source();
|
|
if (result && result.length > 0) {
|
|
title = result;
|
|
break;
|
|
}
|
|
} catch (e) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Extract description
|
|
let description = "";
|
|
const descriptionSources = [
|
|
() => document.querySelector('meta[property="og:description"]')?.getAttribute('content'),
|
|
() => document.querySelector('meta[name="description"]')?.getAttribute('content'),
|
|
() => document.querySelector('meta[name="twitter:description"]')?.getAttribute('content'),
|
|
() => document.querySelector('[itemprop="description"]')?.textContent?.trim()
|
|
];
|
|
|
|
for (const source of descriptionSources) {
|
|
try {
|
|
const result = source();
|
|
if (result && result.length > 0) {
|
|
description = result;
|
|
break;
|
|
}
|
|
} catch (e) {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Comprehensive content selectors with priorities
|
|
const contentSelectors = [
|
|
// Schema.org and structured data
|
|
'[itemtype*="Article"] [itemprop="articleBody"]',
|
|
'[itemtype*="NewsArticle"] [itemprop="articleBody"]',
|
|
'[itemtype*="BlogPosting"] [itemprop="articleBody"]',
|
|
|
|
// High-priority semantic selectors
|
|
'article[role="main"]',
|
|
'main article',
|
|
'[role="main"] article',
|
|
'article',
|
|
|
|
// Common CMS and platform selectors
|
|
'.post-content', '.entry-content', '.article-content', '.content-area',
|
|
'.article-body', '.post-body', '.entry-body', '.story-body',
|
|
'.main-content', '.primary-content', '.page-content',
|
|
'.news-content', '.blog-content', '.editorial-content',
|
|
|
|
// WordPress specific
|
|
'.wp-content', '.entry', '.post',
|
|
|
|
// Medium, Substack, Ghost
|
|
'.section-content', '.postArticle-content', '.post-full-content',
|
|
'.markup', '.section--body', '.section-divider + .section-content',
|
|
|
|
// Japanese sites specific
|
|
'.honbun', '.main_text', '.article_body', '.news_body',
|
|
'.entry_text', '.blog_text', '.content_text',
|
|
'.kiji', '.news', '.article',
|
|
|
|
// Generic semantic HTML5
|
|
'main', '[role="main"]',
|
|
|
|
// ID-based selectors
|
|
'#content', '#main', '#article', '#post', '#entry',
|
|
'#main-content', '#primary', '#content-area',
|
|
|
|
// Class-based common patterns
|
|
'.content', '.main', '.wrapper', '.container',
|
|
|
|
// Fallbacks
|
|
'body'
|
|
];
|
|
|
|
// Function to calculate content quality score
|
|
function calculateContentScore(element: Element): number {
|
|
if (!element) return 0;
|
|
|
|
const text = element.textContent || '';
|
|
if (text.length < 100) return 0;
|
|
|
|
let score = 0;
|
|
|
|
// Base score from text length (diminishing returns)
|
|
score += Math.min(text.length / 100, 50);
|
|
|
|
// Paragraph density
|
|
const paragraphs = element.querySelectorAll('p');
|
|
const avgParagraphLength = paragraphs.length > 0 ?
|
|
Array.from(paragraphs).reduce((sum, p) => sum + (p.textContent?.length || 0), 0) / paragraphs.length : 0;
|
|
|
|
if (avgParagraphLength > 100) score += 20;
|
|
if (paragraphs.length > 3) score += 10;
|
|
|
|
// Link density penalty (articles shouldn't be mostly links)
|
|
const links = element.querySelectorAll('a');
|
|
const linkText = Array.from(links).reduce((sum, link) => sum + (link.textContent?.length || 0), 0);
|
|
const linkDensity = text.length > 0 ? linkText / text.length : 0;
|
|
if (linkDensity < 0.2) score += 15;
|
|
else if (linkDensity < 0.4) score += 5;
|
|
else score -= 10;
|
|
|
|
// Bonus for article-like structure
|
|
if (element.tagName === 'ARTICLE') score += 25;
|
|
if (element.getAttribute('role') === 'main') score += 20;
|
|
if (element.querySelector('h1, h2, h3')) score += 10;
|
|
|
|
// Bonus for semantic elements
|
|
const semanticElements = element.querySelectorAll('p, h1, h2, h3, h4, h5, h6, blockquote, ul, ol');
|
|
if (semanticElements.length > 5) score += 15;
|
|
|
|
// Penalty for too many images without text
|
|
const images = element.querySelectorAll('img');
|
|
if (images.length > text.length / 500) score -= 5;
|
|
|
|
// Penalty for navigation-like content
|
|
const navWords = ['メニュー', 'ナビ', 'カテゴリ', 'タグ', 'menu', 'navigation', 'nav', 'sidebar'];
|
|
const className = element.className.toLowerCase();
|
|
const id = element.id.toLowerCase();
|
|
if (navWords.some(word => className.includes(word) || id.includes(word))) {
|
|
score -= 20;
|
|
}
|
|
|
|
return Math.max(score, 0);
|
|
}
|
|
|
|
// Function to clean and normalize text
|
|
function cleanText(text: string): string {
|
|
return text
|
|
.replace(/\s+/g, ' ') // Normalize whitespace
|
|
.replace(/\n\s*\n\s*\n/g, '\n\n') // Reduce excessive line breaks
|
|
.replace(/^\s+|\s+$/g, '') // Trim
|
|
.replace(/[\u200B-\u200D\uFEFF]/g, '') // Remove zero-width characters
|
|
.trim();
|
|
}
|
|
|
|
// Collect and score all content candidates
|
|
const candidates: ContentCandidate[] = [];
|
|
|
|
for (const selector of contentSelectors) {
|
|
try {
|
|
const elements = document.querySelectorAll(selector);
|
|
elements.forEach((element, index) => {
|
|
const text = element.textContent || '';
|
|
if (text.length > 200) { // Minimum content threshold
|
|
const score = calculateContentScore(element);
|
|
candidates.push({
|
|
element,
|
|
score,
|
|
content: cleanText(text),
|
|
selector: `${selector}[${index}]`
|
|
});
|
|
}
|
|
});
|
|
} catch (e) {
|
|
// Skip invalid selectors
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Sort candidates by score (highest first)
|
|
candidates.sort((a, b) => b.score - a.score);
|
|
|
|
console.log(`Found ${candidates.length} content candidates`);
|
|
if (candidates.length > 0) {
|
|
console.log(`Best candidate score: ${candidates[0]!.score}, selector: ${candidates[0]!.selector}`);
|
|
}
|
|
|
|
// Get the best content
|
|
let content = "";
|
|
if (candidates.length > 0) {
|
|
content = candidates[0]!.content;
|
|
|
|
// If the best candidate is still short, try combining top candidates
|
|
if (content.length < 500 && candidates.length > 1) {
|
|
const topCandidates = candidates.slice(0, 3).filter(c => c.score > 10);
|
|
const combinedContent = topCandidates.map(c => c.content).join('\n\n');
|
|
if (combinedContent.length > content.length) {
|
|
content = cleanText(combinedContent);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Fallback strategies if still no good content
|
|
if (!content || content.length < 200) {
|
|
// Try paragraph aggregation
|
|
const paragraphs = Array.from(document.querySelectorAll('p'))
|
|
.map(p => p.textContent?.trim() || '')
|
|
.filter(p => p.length > 50)
|
|
.filter(p => {
|
|
// Filter out likely navigation/boilerplate paragraphs
|
|
const lowerP = p.toLowerCase();
|
|
return !lowerP.includes('cookie') &&
|
|
!lowerP.includes('privacy') &&
|
|
!lowerP.includes('terms of service') &&
|
|
!lowerP.includes('subscribe') &&
|
|
!lowerP.includes('newsletter');
|
|
});
|
|
|
|
if (paragraphs.length > 0) {
|
|
content = cleanText(paragraphs.join('\n\n'));
|
|
}
|
|
}
|
|
|
|
// Final fallback: structured data
|
|
if (!content || content.length < 200) {
|
|
try {
|
|
const jsonLd = document.querySelector('script[type="application/ld+json"]');
|
|
if (jsonLd) {
|
|
const data = JSON.parse(jsonLd.textContent || '{}');
|
|
if (data.articleBody) {
|
|
content = cleanText(data.articleBody);
|
|
} else if (data.text) {
|
|
content = cleanText(data.text);
|
|
}
|
|
}
|
|
} catch (e) {
|
|
// Ignore JSON parsing errors
|
|
}
|
|
}
|
|
|
|
console.log(`Final content length: ${content.length} characters`);
|
|
return { title, content, description };
|
|
});
|
|
|
|
// Validate extracted content with more lenient threshold
|
|
if (!extractedData.content || extractedData.content.length < 100) {
|
|
// Try one more extraction attempt with relaxed criteria
|
|
const fallbackData = await page.evaluate(() => {
|
|
// Last resort: extract all text from body, excluding common noise
|
|
const body = document.body;
|
|
if (body) {
|
|
// Clone body to avoid modifying original
|
|
const bodyClone = body.cloneNode(true) as Element;
|
|
|
|
// Remove noise elements from clone
|
|
const noiseSelectors = [
|
|
'script', 'style', 'nav', 'header', 'footer', 'aside',
|
|
'.ad', '.ads', '.advertisement', '[class*="ad-"]',
|
|
'.menu', '.navigation', '.sidebar', '.social',
|
|
'.cookie', '.popup', '.modal'
|
|
];
|
|
|
|
noiseSelectors.forEach(selector => {
|
|
const elements = bodyClone.querySelectorAll(selector);
|
|
elements.forEach(el => el.remove());
|
|
});
|
|
|
|
const text = bodyClone.textContent || '';
|
|
return text.replace(/\s+/g, ' ').trim();
|
|
}
|
|
return '';
|
|
});
|
|
|
|
if (fallbackData && fallbackData.length > 200) {
|
|
extractedData.content = fallbackData;
|
|
} else {
|
|
return {
|
|
title: extractedData.title || '',
|
|
content: extractedData.content || "",
|
|
description: extractedData.description || '',
|
|
success: false,
|
|
error: `Insufficient content extracted (${extractedData.content?.length || 0} characters)`,
|
|
};
|
|
}
|
|
}
|
|
|
|
// Limit content length to avoid token limits
|
|
const maxLength = 50000;
|
|
let content = extractedData.content;
|
|
if (content.length > maxLength) {
|
|
content = content.substring(0, maxLength) + "...";
|
|
}
|
|
|
|
console.log(`Successfully extracted content: ${content.length} characters`);
|
|
return {
|
|
title: extractedData.title,
|
|
content,
|
|
description: extractedData.description,
|
|
success: true,
|
|
};
|
|
} catch (error) {
|
|
console.error(`Content extraction attempt failed:`, error);
|
|
throw error; // Let retry logic handle this
|
|
} finally {
|
|
if (page) {
|
|
try {
|
|
await page.close();
|
|
} catch (closeError) {
|
|
console.warn('Failed to close page:', closeError);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
export async function extractArticleContent(
|
|
url: string,
|
|
): Promise<ExtractedContent> {
|
|
console.log(`Starting content extraction for: ${url}`);
|
|
|
|
try {
|
|
return await extractWithRetry(url);
|
|
} catch (error) {
|
|
console.error(`Content extraction failed after all retries for ${url}:`, error);
|
|
|
|
// Check if this is a Puppeteer launch/browser failure that should trigger fallback
|
|
const shouldUseFallback = error instanceof Error && (
|
|
error.message.includes('TimeoutError') ||
|
|
error.message.includes('Timed out after') ||
|
|
error.message.includes('waiting for the WS endpoint URL') ||
|
|
error.message.includes('Browser closed') ||
|
|
error.message.includes('Target closed') ||
|
|
error.message.includes('Session closed') ||
|
|
error.message.includes('Protocol error') ||
|
|
error.message.includes('Connection terminated') ||
|
|
error.message.includes('spawn') || // Process spawn errors
|
|
error.message.includes('ECONNRESET') ||
|
|
error.message.includes('ECONNREFUSED') ||
|
|
error.message.includes('ENOTFOUND')
|
|
);
|
|
|
|
if (shouldUseFallback) {
|
|
console.log(`Puppeteer failed, trying fetch fallback for ${url}`);
|
|
try {
|
|
const fallbackResult = await extractWithFetchFallback(url);
|
|
if (fallbackResult.success) {
|
|
console.log(`Fetch fallback succeeded for ${url}`);
|
|
return fallbackResult;
|
|
}
|
|
console.log(`Fetch fallback also failed for ${url}:`, fallbackResult.error);
|
|
} catch (fallbackError) {
|
|
console.error(`Fetch fallback threw error for ${url}:`, fallbackError);
|
|
}
|
|
}
|
|
|
|
// Provide more specific error messages
|
|
let errorMessage = "Unknown error occurred";
|
|
if (error instanceof Error) {
|
|
if (error.message.includes('ERR_SOCKET_NOT_CONNECTED')) {
|
|
errorMessage = "Network connection failed - server may be unreachable";
|
|
} else if (error.message.includes('ERR_CONNECTION_REFUSED')) {
|
|
errorMessage = "Connection refused by server";
|
|
} else if (error.message.includes('ERR_NAME_NOT_RESOLVED')) {
|
|
errorMessage = "DNS resolution failed - domain may not exist";
|
|
} else if (error.message.includes('ERR_TIMED_OUT')) {
|
|
errorMessage = "Request timed out - server too slow";
|
|
} else if (error.message.includes('HTTP 4')) {
|
|
errorMessage = `Client error: ${error.message}`;
|
|
} else if (error.message.includes('HTTP 5')) {
|
|
errorMessage = `Server error: ${error.message}`;
|
|
} else if (error.message.includes('TimeoutError')) {
|
|
errorMessage = "Puppeteer browser launch timeout - both Puppeteer and fetch fallback failed";
|
|
} else {
|
|
errorMessage = error.message;
|
|
}
|
|
}
|
|
|
|
return {
|
|
title: "",
|
|
content: "",
|
|
description: "",
|
|
success: false,
|
|
error: errorMessage,
|
|
};
|
|
}
|
|
}
|
|
|
|
export async function enhanceArticleContent(
|
|
_originalTitle: string,
|
|
originalLink: string,
|
|
originalContent?: string,
|
|
originalDescription?: string,
|
|
): Promise<{ content?: string; description?: string }> {
|
|
// If we already have substantial content, use it
|
|
const existingContent = originalContent || originalDescription || "";
|
|
if (existingContent.length > 500) {
|
|
return {
|
|
content: originalContent,
|
|
description: originalDescription,
|
|
};
|
|
}
|
|
|
|
// Try to extract content from the URL
|
|
const extracted = await extractArticleContent(originalLink);
|
|
|
|
if (extracted.success && extracted.content) {
|
|
return {
|
|
content: extracted.content,
|
|
description: extracted.description || originalDescription,
|
|
};
|
|
}
|
|
|
|
// Return original content if extraction failed
|
|
return {
|
|
content: originalContent,
|
|
description: originalDescription,
|
|
};
|
|
}
|