diff --git a/src/modules/notifications/services/notification-queue.service.ts b/src/modules/notifications/services/notification-queue.service.ts index d556c51..bb6e087 100644 --- a/src/modules/notifications/services/notification-queue.service.ts +++ b/src/modules/notifications/services/notification-queue.service.ts @@ -74,18 +74,29 @@ export class NotificationQueueService { async addBulkSmsNotifications(jobs: SmsNotificationQueueJob[]): Promise { try { - const queueJobs = jobs.map(job => ({ - name: 'sms-notification', - data: job, - opts: { - jobId: `${job.templateId}-${this.generateRandomInt()}`, - attempts: 3, - backoff: { - type: 'exponential', - delay: 2000, + const queueJobs = jobs.map((job, index) => { + const recipientId = + 'adminId' in job.recipient ? job.recipient.adminId : job.recipient.userId; + return { + name: 'sms-notification', + data: job, + opts: { + jobId: `sms-${job.templateId}-${recipientId}-${index}-${Date.now()}`, + attempts: 3, + backoff: { + type: 'exponential', + delay: 2000, + }, + removeOnComplete: { + age: 24 * 3600, + count: 1000, + }, + removeOnFail: { + age: 7 * 24 * 3600, + }, }, - }, - })); + }; + }); await this.smsQueue.addBulk(queueJobs); this.logger.log(`Added ${jobs.length} SMS notification jobs to queue`); diff --git a/src/modules/orders/crone/order.crone.ts b/src/modules/orders/crone/order.crone.ts index 0f10501..f7064c5 100644 --- a/src/modules/orders/crone/order.crone.ts +++ b/src/modules/orders/crone/order.crone.ts @@ -1,6 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { EntityManager } from '@mikro-orm/postgresql'; +import { DeadlockException } from '@mikro-orm/core'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { Payment } from '../../payments/entities/payment.entity'; import { PaymentMethodEnum, PaymentStatusEnum } from '../../payments/interface/payment'; @@ -8,6 +9,9 @@ import { OrderStatus } from '../interface/order.interface'; import { Order } from '../entities/order.entity'; import { OrderStatusChangedEvent } from '../events/order.events'; +const MAX_DEADLOCK_RETRIES = 3; +const DEADLOCK_RETRY_DELAY_MS = 100; + @Injectable() export class OrdersCrone { private readonly logger = new Logger(OrdersCrone.name); @@ -23,59 +27,88 @@ export class OrdersCrone { timeZone: 'UTC', }) async cancelOldOnlinePendingOrders() { - try { - const cutoff = new Date(Date.now() - 15 * 60 * 1000); - - this.logger.debug('Searching for pending online payments older than 15 minutes'); - - const payments = await this.em.find( - Payment, - { - method: PaymentMethodEnum.Online, - status: PaymentStatusEnum.Pending, - createdAt: { $lte: cutoff }, - }, - { populate: ['order', 'order.items', 'order.items.variant'] }, - ); - - if (!payments || payments.length === 0) { + let lastError: Error | null = null; + for (let attempt = 1; attempt <= MAX_DEADLOCK_RETRIES; attempt++) { + try { + await this.runCancelOldOnlinePendingOrders(); return; - } - - this.logger.log(`Found ${payments.length} stale pending online payments`); - - for (const p of payments) { - try { - await this.em.transactional(async em => { - // reload inside transaction to avoid concurrency issues - const payment = await em.findOne( - Payment, - { id: p.id }, - { populate: ['order', 'order.items', 'order.items.variant'] }, - ); - if (!payment) return; - if (payment.status !== PaymentStatusEnum.Pending) return; - - payment.status = PaymentStatusEnum.Failed; - payment.failedAt = new Date(); - - if (payment.order) { - payment.order.status = OrderStatus.CANCELED; - } - - em.persist(payment); - if (payment.order) em.persist(payment.order); - await em.flush(); - this.logger.log( - `Marked payment ${payment.id} and order ${payment.order?.id} as failed`, - ); - }); - } catch (err) { - this.logger.error(`Error processing payment ${p.id}: ${err.message}`, err.stack); + } catch (err) { + lastError = err; + const isDeadlock = + err instanceof DeadlockException || + (err as Error).message?.includes('deadlock detected'); + if (!isDeadlock || attempt === MAX_DEADLOCK_RETRIES) { + this.logger.error(`OrdersCrone failed: ${(err as Error).message}`, (err as Error).stack); + throw err; } + const delay = DEADLOCK_RETRY_DELAY_MS * Math.pow(2, attempt - 1); + this.logger.warn( + `OrdersCrone deadlock (attempt ${attempt}/${MAX_DEADLOCK_RETRIES}), retrying in ${delay}ms`, + ); + await this.sleep(delay); + } + } + if (lastError) { + this.logger.error(`OrdersCrone failed after ${MAX_DEADLOCK_RETRIES} retries`, lastError.stack); + throw lastError; + } + } + + private sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); + } + + private async runCancelOldOnlinePendingOrders(): Promise { + const cutoff = new Date(Date.now() - 15 * 60 * 1000); + + this.logger.debug('Searching for pending online payments older than 15 minutes'); + + // Minimal query: only payment ids, no joins – reduces lock contention and deadlock risk + const payments = await this.em.find( + Payment, + { + method: PaymentMethodEnum.Online, + status: PaymentStatusEnum.Pending, + createdAt: { $lte: cutoff }, + }, + { fields: ['id'] }, + ); + + if (!payments || payments.length === 0) { + return; + } + + this.logger.log(`Found ${payments.length} stale pending online payments`); + + for (const p of payments) { + try { + await this.em.transactional(async em => { + // reload inside transaction to avoid concurrency issues + const payment = await em.findOne( + Payment, + { id: p.id }, + { populate: ['order', 'order.items', 'order.items.variant'] }, + ); + if (!payment) return; + if (payment.status !== PaymentStatusEnum.Pending) return; + + payment.status = PaymentStatusEnum.Failed; + payment.failedAt = new Date(); + + if (payment.order) { + payment.order.status = OrderStatus.CANCELED; + } + + em.persist(payment); + if (payment.order) em.persist(payment.order); + await em.flush(); + this.logger.log( + `Marked payment ${payment.id} and order ${payment.order?.id} as failed`, + ); + }); + } catch (err) { + this.logger.error(`Error processing payment ${p.id}: ${(err as Error).message}`, (err as Error).stack); } - } catch (err) { - this.logger.error(`OrdersCrone failed: ${err.message}`, err.stack); } }