Initial commit of the Mado Motor NestJS API.
This commit is contained in:
@@ -0,0 +1,320 @@
|
||||
/**
|
||||
* Download images from legacy Parmin storage URLs and re-upload to our Parspack S3.
|
||||
* Updates Product/BlogPost imageUrl + imageKey, and rewrites URLs inside blog bodies.
|
||||
*
|
||||
* Usage:
|
||||
* npx ts-node -r tsconfig-paths/register scripts/migrate-images-to-s3.ts
|
||||
*/
|
||||
import { createHash, randomUUID } from 'crypto'
|
||||
import { extname } from 'path'
|
||||
import { PrismaClient } from '@prisma/client'
|
||||
import { config as loadEnv } from 'dotenv'
|
||||
import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'
|
||||
import {
|
||||
blogMediaKey,
|
||||
productMainKey,
|
||||
} from '../src/storage/storage-keys'
|
||||
|
||||
loadEnv()
|
||||
|
||||
const prisma = new PrismaClient()
|
||||
|
||||
const LEGACY_HOST_RE =
|
||||
/madomotor\.sas\.amin\.parminstorage\.ir|sas\.amin\.parminstorage\.ir/i
|
||||
|
||||
const EXT_MIME: Record<string, string> = {
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.png': 'image/png',
|
||||
'.webp': 'image/webp',
|
||||
'.gif': 'image/gif',
|
||||
}
|
||||
|
||||
function requireEnv(name: string): string {
|
||||
const v = process.env[name]
|
||||
if (!v) throw new Error(`Missing env ${name}`)
|
||||
return v
|
||||
}
|
||||
|
||||
function publicUrlBase(): string {
|
||||
return requireEnv('S3_PUBLIC_URL').replace(/\/$/, '')
|
||||
}
|
||||
|
||||
function isAlreadyOurs(url: string | null | undefined): boolean {
|
||||
if (!url) return false
|
||||
return url.startsWith(publicUrlBase())
|
||||
}
|
||||
|
||||
function isLegacyUrl(url: string): boolean {
|
||||
try {
|
||||
const u = new URL(url)
|
||||
return LEGACY_HOST_RE.test(u.hostname)
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function guessExt(url: string, contentType: string | null): string {
|
||||
const pathPart = decodeURIComponent(new URL(url).pathname)
|
||||
const fromUrl = extname(pathPart).toLowerCase()
|
||||
if (fromUrl && fromUrl.length <= 5) return fromUrl
|
||||
if (contentType?.includes('png')) return '.png'
|
||||
if (contentType?.includes('webp')) return '.webp'
|
||||
if (contentType?.includes('gif')) return '.gif'
|
||||
return '.jpg'
|
||||
}
|
||||
|
||||
function mimeFor(ext: string, contentType: string | null): string {
|
||||
if (contentType && contentType.startsWith('image/')) {
|
||||
return contentType.split(';')[0].trim()
|
||||
}
|
||||
return EXT_MIME[ext] ?? 'image/jpeg'
|
||||
}
|
||||
|
||||
function extractUrls(text: string): string[] {
|
||||
const found = new Set<string>()
|
||||
const patterns = [
|
||||
/https?:\/\/[^\s"'<>]+/g,
|
||||
/src=["'](https?:\/\/[^"']+)["']/gi,
|
||||
]
|
||||
for (const re of patterns) {
|
||||
for (const match of text.matchAll(re)) {
|
||||
const url = (match[1] ?? match[0]).replace(/&/g, '&')
|
||||
found.add(url)
|
||||
}
|
||||
}
|
||||
return [...found]
|
||||
}
|
||||
|
||||
type Kind = 'product' | 'blog' | 'blog-body'
|
||||
|
||||
function encodeFetchUrl(url: string): string {
|
||||
try {
|
||||
const u = new URL(url)
|
||||
// Encode each path segment so Persian filenames download reliably
|
||||
u.pathname = u.pathname
|
||||
.split('/')
|
||||
.map((seg) => {
|
||||
if (!seg) return seg
|
||||
try {
|
||||
return encodeURIComponent(decodeURIComponent(seg))
|
||||
} catch {
|
||||
return encodeURIComponent(seg)
|
||||
}
|
||||
})
|
||||
.join('/')
|
||||
return u.toString()
|
||||
} catch {
|
||||
return encodeURI(url)
|
||||
}
|
||||
}
|
||||
|
||||
async function download(url: string): Promise<{
|
||||
buffer: Buffer
|
||||
contentType: string | null
|
||||
}> {
|
||||
const candidates = [url, encodeFetchUrl(url)]
|
||||
let lastError: Error | null = null
|
||||
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
const res = await fetch(candidate, {
|
||||
redirect: 'follow',
|
||||
headers: {
|
||||
'User-Agent': 'mado-motor-image-migrator/1.0',
|
||||
Accept: 'image/*,*/*',
|
||||
},
|
||||
})
|
||||
if (!res.ok) {
|
||||
lastError = new Error(`HTTP ${res.status} for ${candidate}`)
|
||||
continue
|
||||
}
|
||||
const contentType = res.headers.get('content-type')
|
||||
const buffer = Buffer.from(await res.arrayBuffer())
|
||||
if (buffer.length < 32) {
|
||||
lastError = new Error(`Tiny response (${buffer.length}b) for ${candidate}`)
|
||||
continue
|
||||
}
|
||||
return { buffer, contentType }
|
||||
} catch (err) {
|
||||
lastError = err instanceof Error ? err : new Error(String(err))
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError ?? new Error(`Failed to download ${url}`)
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const endpoint = requireEnv('S3_ENDPOINT')
|
||||
const bucket = requireEnv('S3_BUCKET')
|
||||
const base = publicUrlBase()
|
||||
|
||||
const client = new S3Client({
|
||||
endpoint,
|
||||
region: process.env.S3_REGION || 'us-east-1',
|
||||
forcePathStyle: (process.env.S3_FORCE_PATH_STYLE || 'true') === 'true',
|
||||
credentials: {
|
||||
accessKeyId: requireEnv('S3_ACCESS_KEY_ID'),
|
||||
secretAccessKey: requireEnv('S3_SECRET_ACCESS_KEY'),
|
||||
},
|
||||
})
|
||||
|
||||
const products = await prisma.product.findMany({
|
||||
select: { id: true, imageUrl: true, imageKey: true },
|
||||
})
|
||||
const blogs = await prisma.blogPost.findMany({
|
||||
select: { id: true, imageUrl: true, imageKey: true, body: true },
|
||||
})
|
||||
|
||||
type Job = { url: string; kind: Kind; entityId?: string }
|
||||
const jobs: Job[] = []
|
||||
const seen = new Set<string>()
|
||||
|
||||
const enqueue = (url: string, kind: Kind, entityId?: string) => {
|
||||
const clean = url.replace(/[),.;]+$/, '')
|
||||
if (!isLegacyUrl(clean) || isAlreadyOurs(clean)) return
|
||||
if (seen.has(clean)) return
|
||||
seen.add(clean)
|
||||
jobs.push({ url: clean, kind, entityId })
|
||||
}
|
||||
|
||||
for (const p of products) {
|
||||
if (p.imageUrl) enqueue(p.imageUrl, 'product', p.id)
|
||||
}
|
||||
for (const b of blogs) {
|
||||
if (b.imageUrl) enqueue(b.imageUrl, 'blog', b.id)
|
||||
for (const u of extractUrls(b.body || '')) {
|
||||
enqueue(u, 'blog-body')
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Found ${jobs.length} unique legacy URLs to migrate`)
|
||||
|
||||
const urlMap = new Map<string, { url: string; key: string }>()
|
||||
let ok = 0
|
||||
let failed = 0
|
||||
|
||||
for (const [i, job] of jobs.entries()) {
|
||||
const label = `[${i + 1}/${jobs.length}]`
|
||||
try {
|
||||
const { buffer, contentType } = await download(job.url)
|
||||
const ext = guessExt(job.url, contentType)
|
||||
const fileName = `${randomUUID()}${ext}`
|
||||
const key =
|
||||
job.kind === 'product'
|
||||
? productMainKey(fileName)
|
||||
: blogMediaKey(
|
||||
job.kind === 'blog-body' ? `body/${fileName}` : fileName,
|
||||
)
|
||||
const contentTypeFinal = mimeFor(ext, contentType)
|
||||
|
||||
await client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: key,
|
||||
Body: buffer,
|
||||
ContentType: contentTypeFinal,
|
||||
}),
|
||||
)
|
||||
|
||||
const publicUrl = `${base}/${key}`
|
||||
urlMap.set(job.url, { url: publicUrl, key })
|
||||
|
||||
// also map common encoded/decoded variants
|
||||
try {
|
||||
urlMap.set(decodeURI(job.url), { url: publicUrl, key })
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
urlMap.set(encodeURI(decodeURIComponent(job.url)), {
|
||||
url: publicUrl,
|
||||
key,
|
||||
})
|
||||
|
||||
ok += 1
|
||||
const hash = createHash('sha1').update(buffer).digest('hex').slice(0, 8)
|
||||
console.log(
|
||||
`${label} OK ${job.kind} ${buffer.length}b ${hash} → ${key}`,
|
||||
)
|
||||
} catch (err) {
|
||||
failed += 1
|
||||
console.error(
|
||||
`${label} FAIL ${job.url}`,
|
||||
err instanceof Error ? err.message : err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Update products
|
||||
let productsUpdated = 0
|
||||
for (const p of products) {
|
||||
if (!p.imageUrl) continue
|
||||
const mapped = urlMap.get(p.imageUrl) || urlMap.get(p.imageUrl.replace(/[),.;]+$/, ''))
|
||||
if (!mapped) continue
|
||||
await prisma.product.update({
|
||||
where: { id: p.id },
|
||||
data: { imageUrl: mapped.url, imageKey: mapped.key },
|
||||
})
|
||||
productsUpdated += 1
|
||||
}
|
||||
|
||||
// Update blogs (cover + body rewrite)
|
||||
let blogsUpdated = 0
|
||||
for (const b of blogs) {
|
||||
let body = b.body || ''
|
||||
let changed = false
|
||||
|
||||
for (const [oldUrl, mapped] of urlMap) {
|
||||
if (body.includes(oldUrl)) {
|
||||
body = body.split(oldUrl).join(mapped.url)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
|
||||
// also rewrite any remaining legacy host URLs that we mapped by pathname
|
||||
const stillLegacy = extractUrls(body).filter(isLegacyUrl)
|
||||
for (const u of stillLegacy) {
|
||||
const clean = u.replace(/[),.;]+$/, '')
|
||||
const mapped = urlMap.get(clean)
|
||||
if (mapped && body.includes(u)) {
|
||||
body = body.split(u).join(mapped.url)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
|
||||
const coverMapped = b.imageUrl
|
||||
? urlMap.get(b.imageUrl) ||
|
||||
urlMap.get(b.imageUrl.replace(/[),.;]+$/, ''))
|
||||
: undefined
|
||||
|
||||
if (!changed && !coverMapped) continue
|
||||
|
||||
await prisma.blogPost.update({
|
||||
where: { id: b.id },
|
||||
data: {
|
||||
...(coverMapped
|
||||
? { imageUrl: coverMapped.url, imageKey: coverMapped.key }
|
||||
: {}),
|
||||
...(changed ? { body } : {}),
|
||||
},
|
||||
})
|
||||
blogsUpdated += 1
|
||||
}
|
||||
|
||||
console.log('Done:', {
|
||||
migrated: ok,
|
||||
failed,
|
||||
productsUpdated,
|
||||
blogsUpdated,
|
||||
publicBase: base,
|
||||
})
|
||||
}
|
||||
|
||||
main()
|
||||
.catch((err) => {
|
||||
console.error(err)
|
||||
process.exitCode = 1
|
||||
})
|
||||
.finally(async () => {
|
||||
await prisma.$disconnect()
|
||||
})
|
||||
Reference in New Issue
Block a user