diff --git a/src/categories/category-technical-form-ai.service.ts b/src/categories/category-technical-form-ai.service.ts index 1239a55..a3505af 100644 --- a/src/categories/category-technical-form-ai.service.ts +++ b/src/categories/category-technical-form-ai.service.ts @@ -10,6 +10,7 @@ import { requestAiJsonCompletion, resolveAiProvider, } from '../common/ai-provider.util'; +import { loadReferencePagesFromText } from '../common/ai-reference-pages.util'; import { PrismaService } from '../prisma/prisma.service'; import { CategoryTechnicalFormService } from './category-technical-form.service'; import { @@ -86,6 +87,14 @@ export class CategoryTechnicalFormAiService { existingFieldCount: number; }): Promise { const provider = resolveAiProvider(this.config); + const referencePages = await loadReferencePagesFromText(input.hint ?? ''); + const usableReferences = referencePages.filter((page) => page.text); + + if (referencePages.length > 0 && usableReferences.length === 0) { + throw new BadRequestException( + 'Could not load the product page URL from the prompt. Check the link and try again.', + ); + } const systemPrompt = `You design technical specification forms for e-commerce product categories on an Iranian marketplace. Return ONLY valid JSON with this shape: @@ -110,7 +119,11 @@ Rules: - Labels must be unique and human-readable. - Write labels and option text in the same language as the user hint. If the hint is Persian (Farsi), use Persian labels and options; otherwise use English. - Do not include price, stock, SKU, or warranty period fields. -- Prefer metric units common in Iran when relevant.`; +- Prefer metric units common in Iran when relevant. +- When referencePages are provided, treat them as the source of truth: build fields that match the real specs / attributes shown on those pages (capacity, power, dimensions, materials, programs, colors, etc.). +- Prefer the main product specs near the product title; ignore conflicting related-product / upsell blocks. +- Do NOT invent field topics from memory when a reference page was provided — mirror what the page actually documents. +- Convert units carefully in labels when helpful (e.g. dimensions in cm if the page lists mm).`; const userPrompt = JSON.stringify({ category: input.categoryName, @@ -120,11 +133,23 @@ Rules: input.existingFieldCount > 0 ? 'Category already has a form; suggest a fresh complete replacement set.' : 'Category has no form yet.', + referencePages: usableReferences.map((page) => ({ + url: page.url, + text: page.text, + })), + referenceFetchErrors: referencePages + .filter((page) => !page.text) + .map((page) => ({ url: page.url, error: page.error ?? 'unknown' })), }); let content: string; try { - content = await requestAiJsonCompletion(provider, systemPrompt, userPrompt); + content = await requestAiJsonCompletion( + provider, + systemPrompt, + userPrompt, + usableReferences.length > 0 ? 0.2 : 0.4, + ); } catch (err) { const message = err instanceof Error ? err.message : 'AI request failed'; throw new BadRequestException(message); diff --git a/src/common/ai-reference-pages.util.ts b/src/common/ai-reference-pages.util.ts new file mode 100644 index 0000000..bfacf20 --- /dev/null +++ b/src/common/ai-reference-pages.util.ts @@ -0,0 +1,123 @@ +const URL_RE = /https?:\/\/[^\s<>"'`)\]]+/gi; + +const MAX_URLS = 3; +const MAX_BYTES = 1_500_000; +const MAX_TEXT_CHARS = 12_000; +const FETCH_TIMEOUT_MS = 12_000; + +export function extractHttpUrls(text: string): string[] { + const matches = text.match(URL_RE) ?? []; + const seen = new Set(); + const urls: string[] = []; + + for (const raw of matches) { + const cleaned = raw.replace(/[.,;:!?]+$/g, ''); + if (!cleaned || seen.has(cleaned)) continue; + try { + const parsed = new URL(cleaned); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') continue; + } catch { + continue; + } + seen.add(cleaned); + urls.push(cleaned); + if (urls.length >= MAX_URLS) break; + } + + return urls; +} + +function stripHtmlToText(html: string): string { + const withoutNoise = html + .replace(//gi, ' ') + .replace(//gi, ' ') + .replace(//gi, ' ') + .replace(//g, ' ') + .replace(/<\/(p|div|li|tr|h[1-6]|br|table|section|article)>/gi, '\n') + .replace(/<(br|hr)\s*\/?>/gi, '\n') + .replace(/<[^>]+>/g, ' ') + .replace(/ /gi, ' ') + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/&#(\d+);/g, (_, code) => { + const n = Number(code); + return Number.isFinite(n) ? String.fromCharCode(n) : ' '; + }) + .replace(/\s+\n/g, '\n') + .replace(/\n{3,}/g, '\n\n') + .replace(/[ \t]{2,}/g, ' ') + .trim(); + + return withoutNoise.slice(0, MAX_TEXT_CHARS); +} + +export async function fetchUrlAsPlainText(url: string): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + + try { + const response = await fetch(url, { + method: 'GET', + redirect: 'follow', + signal: controller.signal, + headers: { + Accept: 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8', + 'User-Agent': + 'Mozilla/5.0 (compatible; MeshkeeBot/1.0; +https://meshkee.com)', + }, + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + + const contentType = response.headers.get('content-type') ?? ''; + if ( + contentType && + !/text\/html|application\/xhtml\+xml|text\/plain/i.test(contentType) + ) { + throw new Error(`Unsupported content type: ${contentType}`); + } + + const buffer = Buffer.from(await response.arrayBuffer()); + const truncated = buffer.subarray(0, MAX_BYTES).toString('utf8'); + const text = stripHtmlToText(truncated); + if (!text || text.length < 40) { + throw new Error('Page content was empty'); + } + return text; + } finally { + clearTimeout(timer); + } +} + +export type ReferencePage = { + url: string; + text: string | null; + error?: string; +}; + +/** + * Extract http(s) URLs from a prompt and fetch their readable text for AI grounding. + */ +export async function loadReferencePagesFromText( + text: string, +): Promise { + const urls = extractHttpUrls(text); + if (urls.length === 0) return []; + + const pages = await Promise.all( + urls.map(async (url) => { + try { + const pageText = await fetchUrlAsPlainText(url); + return { url, text: pageText } satisfies ReferencePage; + } catch (err) { + const message = err instanceof Error ? err.message : 'Fetch failed'; + return { url, text: null, error: message } satisfies ReferencePage; + } + }), + ); + + return pages; +} diff --git a/src/products/product-ai.service.ts b/src/products/product-ai.service.ts index 25a964f..d96a815 100644 --- a/src/products/product-ai.service.ts +++ b/src/products/product-ai.service.ts @@ -20,6 +20,7 @@ import { requestAiJsonCompletion, resolveAiProvider, } from '../common/ai-provider.util'; +import { loadReferencePagesFromText } from '../common/ai-reference-pages.util'; import { normalizeAiDescriptionHtml } from '../common/ai-html.util'; type TechnicalFormField = { @@ -324,6 +325,9 @@ Aim for 2–4 short paragraphs and optionally a bullet list.`; })); const provider = resolveAiProvider(this.config); + const referencePages = await loadReferencePagesFromText(prompt); + const usableReferences = referencePages.filter((page) => page.text); + const systemPrompt = `You fill e-commerce product technical specification forms for Iranian stores. Return ONLY valid JSON with this shape: { "technicalValues": { "fieldKey": "value" } } @@ -333,8 +337,11 @@ Rules: - For select fields use one allowed option "value" exactly (not the label). - For multi_select fields use an array of allowed option values. - For text/textarea fields use plain strings in the same language as the field labels / user prompt (usually Persian). -- Prefer factual values for the identified product. Do not invent unrealistic specs. -- Leave a field out if you cannot determine a reasonable value. +- When referencePages are provided, treat them as the source of truth. Copy specs from that page text (capacity, power, dimensions, weight, materials, programs, etc.). +- Convert units carefully (e.g. mm → cm for dimension fields labeled in cm: 279x300x464 mm → 27.9x30x46.4 cm). +- Prefer the product-specific specs near the main product title; ignore unrelated related-products / upsell tables if they conflict. +- Do NOT invent or guess specs from memory when a reference page was provided. +- Leave a field out if the reference does not contain that value. - Follow the user's research brief / instructions when provided.`; const userPrompt = JSON.stringify({ @@ -344,11 +351,24 @@ Rules: nameFa: nameFa || null, }, technicalFields: technicalSchema, + referencePages: usableReferences.map((page) => ({ + url: page.url, + text: page.text, + })), + referenceFetchErrors: referencePages + .filter((page) => !page.text) + .map((page) => ({ url: page.url, error: page.error ?? 'unknown' })), }); + if (referencePages.length > 0 && usableReferences.length === 0) { + throw new BadRequestException( + 'Could not load the product page URL from the prompt. Check the link and try again.', + ); + } + let content: string; try { - content = await requestAiJsonCompletion(provider, systemPrompt, userPrompt, 0.4); + content = await requestAiJsonCompletion(provider, systemPrompt, userPrompt, 0.2); } catch (err) { const message = err instanceof Error ? err.message : 'AI request failed'; throw new BadRequestException(message); @@ -417,6 +437,8 @@ Rules: fields: TechnicalFormField[]; }): Promise { const provider = resolveAiProvider(this.config); + const referencePages = await loadReferencePagesFromText(input.productName); + const usableReferences = referencePages.filter((page) => page.text); const technicalSchema = input.fields.map((field) => ({ key: field.key, @@ -447,6 +469,9 @@ Rules: - The user input is a research brief in ${inputLanguage}. It may include a model/product name, a product page URL, and extra notes — not a final storefront title. - Infer the real product from the brief (and any URL/model clues). Do not copy URLs, markdown, or raw notes into title/nameFa. - Write clean storefront titles: "title" in English, "nameFa" in Persian. Keep brand/model accurate. +- When referencePages are provided, treat them as the source of truth for specs, capacity, power, dimensions, and description facts. +- Prefer the main product specs near the product title; ignore conflicting related-product / upsell blocks. +- Do NOT invent specs from memory when a reference page was provided. - Be factual and realistic for the identified product in the given category. - If technical form fields are provided, fill technicalValues using exact field keys. - For select fields use one allowed option value exactly. @@ -461,11 +486,18 @@ Rules: researchBrief: input.productName, inputLanguage: input.language, technicalFields: technicalSchema, + referencePages: usableReferences.map((page) => ({ + url: page.url, + text: page.text, + })), + referenceFetchErrors: referencePages + .filter((page) => !page.text) + .map((page) => ({ url: page.url, error: page.error ?? 'unknown' })), }); let content: string; try { - content = await requestAiJsonCompletion(provider, systemPrompt, userPrompt); + content = await requestAiJsonCompletion(provider, systemPrompt, userPrompt, 0.3); } catch (err) { const message = err instanceof Error ? err.message : 'AI request failed'; throw new BadRequestException(message);