Fetch product page URLs when generating technical AI content.

Ground product technical fill, product AI create, and category technical form suggestions on page text instead of inventing specs from the URL alone.

Co-authored-by: Cursor <cursoragent@cursor.com>

src/categories/category-technical-form-ai.service.ts

@@ -10,6 +10,7 @@ import {
   requestAiJsonCompletion,
   resolveAiProvider,
 } from '../common/ai-provider.util';
+import { loadReferencePagesFromText } from '../common/ai-reference-pages.util';
 import { PrismaService } from '../prisma/prisma.service';
 import { CategoryTechnicalFormService } from './category-technical-form.service';
 import {
@@ -86,6 +87,14 @@ export class CategoryTechnicalFormAiService {
     existingFieldCount: number;
   }): Promise<TechnicalFormFieldInputDto[]> {
     const provider = resolveAiProvider(this.config);
+    const referencePages = await loadReferencePagesFromText(input.hint ?? '');
+    const usableReferences = referencePages.filter((page) => page.text);
+
+    if (referencePages.length > 0 && usableReferences.length === 0) {
+      throw new BadRequestException(
+        'Could not load the product page URL from the prompt. Check the link and try again.',
+      );
+    }
 
     const systemPrompt = `You design technical specification forms for e-commerce product categories on an Iranian marketplace.
 Return ONLY valid JSON with this shape:
@@ -110,7 +119,11 @@ Rules:
 - Labels must be unique and human-readable.
 - Write labels and option text in the same language as the user hint. If the hint is Persian (Farsi), use Persian labels and options; otherwise use English.
 - Do not include price, stock, SKU, or warranty period fields.
-- Prefer metric units common in Iran when relevant.`;
+- Prefer metric units common in Iran when relevant.
+- When referencePages are provided, treat them as the source of truth: build fields that match the real specs / attributes shown on those pages (capacity, power, dimensions, materials, programs, colors, etc.).
+- Prefer the main product specs near the product title; ignore conflicting related-product / upsell blocks.
+- Do NOT invent field topics from memory when a reference page was provided — mirror what the page actually documents.
+- Convert units carefully in labels when helpful (e.g. dimensions in cm if the page lists mm).`;
 
     const userPrompt = JSON.stringify({
       category: input.categoryName,
@@ -120,11 +133,23 @@ Rules:
         input.existingFieldCount > 0
           ? 'Category already has a form; suggest a fresh complete replacement set.'
           : 'Category has no form yet.',
+      referencePages: usableReferences.map((page) => ({
+        url: page.url,
+        text: page.text,
+      })),
+      referenceFetchErrors: referencePages
+        .filter((page) => !page.text)
+        .map((page) => ({ url: page.url, error: page.error ?? 'unknown' })),
     });
 
     let content: string;
     try {
-      content = await requestAiJsonCompletion(provider, systemPrompt, userPrompt);
+      content = await requestAiJsonCompletion(
+        provider,
+        systemPrompt,
+        userPrompt,
+        usableReferences.length > 0 ? 0.2 : 0.4,
+      );
     } catch (err) {
       const message = err instanceof Error ? err.message : 'AI request failed';
       throw new BadRequestException(message);

src/common/ai-reference-pages.util.ts

@@ -0,0 +1,123 @@
+const URL_RE = /https?:\/\/[^\s<>"'`)\]]+/gi;
+
+const MAX_URLS = 3;
+const MAX_BYTES = 1_500_000;
+const MAX_TEXT_CHARS = 12_000;
+const FETCH_TIMEOUT_MS = 12_000;
+
+export function extractHttpUrls(text: string): string[] {
+  const matches = text.match(URL_RE) ?? [];
+  const seen = new Set<string>();
+  const urls: string[] = [];
+
+  for (const raw of matches) {
+    const cleaned = raw.replace(/[.,;:!?]+$/g, '');
+    if (!cleaned || seen.has(cleaned)) continue;
+    try {
+      const parsed = new URL(cleaned);
+      if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') continue;
+    } catch {
+      continue;
+    }
+    seen.add(cleaned);
+    urls.push(cleaned);
+    if (urls.length >= MAX_URLS) break;
+  }
+
+  return urls;
+}
+
+function stripHtmlToText(html: string): string {
+  const withoutNoise = html
+    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+    .replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ')
+    .replace(/<!--[\s\S]*?-->/g, ' ')
+    .replace(/<\/(p|div|li|tr|h[1-6]|br|table|section|article)>/gi, '\n')
+    .replace(/<(br|hr)\s*\/?>/gi, '\n')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&nbsp;/gi, ' ')
+    .replace(/&amp;/gi, '&')
+    .replace(/&lt;/gi, '<')
+    .replace(/&gt;/gi, '>')
+    .replace(/&#(\d+);/g, (_, code) => {
+      const n = Number(code);
+      return Number.isFinite(n) ? String.fromCharCode(n) : ' ';
+    })
+    .replace(/\s+\n/g, '\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .replace(/[ \t]{2,}/g, ' ')
+    .trim();
+
+  return withoutNoise.slice(0, MAX_TEXT_CHARS);
+}
+
+export async function fetchUrlAsPlainText(url: string): Promise<string> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+
+  try {
+    const response = await fetch(url, {
+      method: 'GET',
+      redirect: 'follow',
+      signal: controller.signal,
+      headers: {
+        Accept: 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8',
+        'User-Agent':
+          'Mozilla/5.0 (compatible; MeshkeeBot/1.0; +https://meshkee.com)',
+      },
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}`);
+    }
+
+    const contentType = response.headers.get('content-type') ?? '';
+    if (
+      contentType &&
+      !/text\/html|application\/xhtml\+xml|text\/plain/i.test(contentType)
+    ) {
+      throw new Error(`Unsupported content type: ${contentType}`);
+    }
+
+    const buffer = Buffer.from(await response.arrayBuffer());
+    const truncated = buffer.subarray(0, MAX_BYTES).toString('utf8');
+    const text = stripHtmlToText(truncated);
+    if (!text || text.length < 40) {
+      throw new Error('Page content was empty');
+    }
+    return text;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export type ReferencePage = {
+  url: string;
+  text: string | null;
+  error?: string;
+};
+
+/**
+ * Extract http(s) URLs from a prompt and fetch their readable text for AI grounding.
+ */
+export async function loadReferencePagesFromText(
+  text: string,
+): Promise<ReferencePage[]> {
+  const urls = extractHttpUrls(text);
+  if (urls.length === 0) return [];
+
+  const pages = await Promise.all(
+    urls.map(async (url) => {
+      try {
+        const pageText = await fetchUrlAsPlainText(url);
+        return { url, text: pageText } satisfies ReferencePage;
+      } catch (err) {
+        const message = err instanceof Error ? err.message : 'Fetch failed';
+        return { url, text: null, error: message } satisfies ReferencePage;
+      }
+    }),
+  );
+
+  return pages;
+}

src/products/product-ai.service.ts

@@ -20,6 +20,7 @@ import {
   requestAiJsonCompletion,
   resolveAiProvider,
 } from '../common/ai-provider.util';
+import { loadReferencePagesFromText } from '../common/ai-reference-pages.util';
 import { normalizeAiDescriptionHtml } from '../common/ai-html.util';
 
 type TechnicalFormField = {
@@ -324,6 +325,9 @@ Aim for 2–4 short paragraphs and optionally a bullet list.`;
     }));
 
     const provider = resolveAiProvider(this.config);
+    const referencePages = await loadReferencePagesFromText(prompt);
+    const usableReferences = referencePages.filter((page) => page.text);
+
     const systemPrompt = `You fill e-commerce product technical specification forms for Iranian stores.
 Return ONLY valid JSON with this shape:
 { "technicalValues": { "fieldKey": "value" } }
@@ -333,8 +337,11 @@ Rules:
 - For select fields use one allowed option "value" exactly (not the label).
 - For multi_select fields use an array of allowed option values.
 - For text/textarea fields use plain strings in the same language as the field labels / user prompt (usually Persian).
-- Prefer factual values for the identified product. Do not invent unrealistic specs.
-- Leave a field out if you cannot determine a reasonable value.
+- When referencePages are provided, treat them as the source of truth. Copy specs from that page text (capacity, power, dimensions, weight, materials, programs, etc.).
+- Convert units carefully (e.g. mm → cm for dimension fields labeled in cm: 279x300x464 mm → 27.9x30x46.4 cm).
+- Prefer the product-specific specs near the main product title; ignore unrelated related-products / upsell tables if they conflict.
+- Do NOT invent or guess specs from memory when a reference page was provided.
+- Leave a field out if the reference does not contain that value.
 - Follow the user's research brief / instructions when provided.`;
 
     const userPrompt = JSON.stringify({
@@ -344,11 +351,24 @@ Rules:
         nameFa: nameFa || null,
       },
       technicalFields: technicalSchema,
+      referencePages: usableReferences.map((page) => ({
+        url: page.url,
+        text: page.text,
+      })),
+      referenceFetchErrors: referencePages
+        .filter((page) => !page.text)
+        .map((page) => ({ url: page.url, error: page.error ?? 'unknown' })),
     });
 
+    if (referencePages.length > 0 && usableReferences.length === 0) {
+      throw new BadRequestException(
+        'Could not load the product page URL from the prompt. Check the link and try again.',
+      );
+    }
+
     let content: string;
     try {
-      content = await requestAiJsonCompletion(provider, systemPrompt, userPrompt, 0.4);
+      content = await requestAiJsonCompletion(provider, systemPrompt, userPrompt, 0.2);
     } catch (err) {
       const message = err instanceof Error ? err.message : 'AI request failed';
       throw new BadRequestException(message);
@@ -417,6 +437,8 @@ Rules:
     fields: TechnicalFormField[];
   }): Promise<AiProductDraft> {
     const provider = resolveAiProvider(this.config);
+    const referencePages = await loadReferencePagesFromText(input.productName);
+    const usableReferences = referencePages.filter((page) => page.text);
 
     const technicalSchema = input.fields.map((field) => ({
       key: field.key,
@@ -447,6 +469,9 @@ Rules:
 - The user input is a research brief in ${inputLanguage}. It may include a model/product name, a product page URL, and extra notes — not a final storefront title.
 - Infer the real product from the brief (and any URL/model clues). Do not copy URLs, markdown, or raw notes into title/nameFa.
 - Write clean storefront titles: "title" in English, "nameFa" in Persian. Keep brand/model accurate.
+- When referencePages are provided, treat them as the source of truth for specs, capacity, power, dimensions, and description facts.
+- Prefer the main product specs near the product title; ignore conflicting related-product / upsell blocks.
+- Do NOT invent specs from memory when a reference page was provided.
 - Be factual and realistic for the identified product in the given category.
 - If technical form fields are provided, fill technicalValues using exact field keys.
 - For select fields use one allowed option value exactly.
@@ -461,11 +486,18 @@ Rules:
       researchBrief: input.productName,
       inputLanguage: input.language,
       technicalFields: technicalSchema,
+      referencePages: usableReferences.map((page) => ({
+        url: page.url,
+        text: page.text,
+      })),
+      referenceFetchErrors: referencePages
+        .filter((page) => !page.text)
+        .map((page) => ({ url: page.url, error: page.error ?? 'unknown' })),
     });
 
     let content: string;
     try {
-      content = await requestAiJsonCompletion(provider, systemPrompt, userPrompt);
+      content = await requestAiJsonCompletion(provider, systemPrompt, userPrompt, 0.3);
     } catch (err) {
       const message = err instanceof Error ? err.message : 'AI request failed';
       throw new BadRequestException(message);