This commit is contained in:
2026-02-14 09:25:26 +03:30
parent 5525634475
commit a2727862bf
2 changed files with 105 additions and 61 deletions
@@ -74,18 +74,29 @@ export class NotificationQueueService {
async addBulkSmsNotifications(jobs: SmsNotificationQueueJob[]): Promise<void> {
try {
const queueJobs = jobs.map(job => ({
name: 'sms-notification',
data: job,
opts: {
jobId: `${job.templateId}-${this.generateRandomInt()}`,
attempts: 3,
backoff: {
type: 'exponential',
delay: 2000,
const queueJobs = jobs.map((job, index) => {
const recipientId =
'adminId' in job.recipient ? job.recipient.adminId : job.recipient.userId;
return {
name: 'sms-notification',
data: job,
opts: {
jobId: `sms-${job.templateId}-${recipientId}-${index}-${Date.now()}`,
attempts: 3,
backoff: {
type: 'exponential',
delay: 2000,
},
removeOnComplete: {
age: 24 * 3600,
count: 1000,
},
removeOnFail: {
age: 7 * 24 * 3600,
},
},
},
}));
};
});
await this.smsQueue.addBulk(queueJobs);
this.logger.log(`Added ${jobs.length} SMS notification jobs to queue`);
+83 -50
View File
@@ -1,6 +1,7 @@
import { Injectable, Logger } from '@nestjs/common';
import { Cron } from '@nestjs/schedule';
import { EntityManager } from '@mikro-orm/postgresql';
import { DeadlockException } from '@mikro-orm/core';
import { EventEmitter2 } from '@nestjs/event-emitter';
import { Payment } from '../../payments/entities/payment.entity';
import { PaymentMethodEnum, PaymentStatusEnum } from '../../payments/interface/payment';
@@ -8,6 +9,9 @@ import { OrderStatus } from '../interface/order.interface';
import { Order } from '../entities/order.entity';
import { OrderStatusChangedEvent } from '../events/order.events';
const MAX_DEADLOCK_RETRIES = 3;
const DEADLOCK_RETRY_DELAY_MS = 100;
@Injectable()
export class OrdersCrone {
private readonly logger = new Logger(OrdersCrone.name);
@@ -23,59 +27,88 @@ export class OrdersCrone {
timeZone: 'UTC',
})
async cancelOldOnlinePendingOrders() {
try {
const cutoff = new Date(Date.now() - 15 * 60 * 1000);
this.logger.debug('Searching for pending online payments older than 15 minutes');
const payments = await this.em.find(
Payment,
{
method: PaymentMethodEnum.Online,
status: PaymentStatusEnum.Pending,
createdAt: { $lte: cutoff },
},
{ populate: ['order', 'order.items', 'order.items.variant'] },
);
if (!payments || payments.length === 0) {
let lastError: Error | null = null;
for (let attempt = 1; attempt <= MAX_DEADLOCK_RETRIES; attempt++) {
try {
await this.runCancelOldOnlinePendingOrders();
return;
}
this.logger.log(`Found ${payments.length} stale pending online payments`);
for (const p of payments) {
try {
await this.em.transactional(async em => {
// reload inside transaction to avoid concurrency issues
const payment = await em.findOne(
Payment,
{ id: p.id },
{ populate: ['order', 'order.items', 'order.items.variant'] },
);
if (!payment) return;
if (payment.status !== PaymentStatusEnum.Pending) return;
payment.status = PaymentStatusEnum.Failed;
payment.failedAt = new Date();
if (payment.order) {
payment.order.status = OrderStatus.CANCELED;
}
em.persist(payment);
if (payment.order) em.persist(payment.order);
await em.flush();
this.logger.log(
`Marked payment ${payment.id} and order ${payment.order?.id} as failed`,
);
});
} catch (err) {
this.logger.error(`Error processing payment ${p.id}: ${err.message}`, err.stack);
} catch (err) {
lastError = err;
const isDeadlock =
err instanceof DeadlockException ||
(err as Error).message?.includes('deadlock detected');
if (!isDeadlock || attempt === MAX_DEADLOCK_RETRIES) {
this.logger.error(`OrdersCrone failed: ${(err as Error).message}`, (err as Error).stack);
throw err;
}
const delay = DEADLOCK_RETRY_DELAY_MS * Math.pow(2, attempt - 1);
this.logger.warn(
`OrdersCrone deadlock (attempt ${attempt}/${MAX_DEADLOCK_RETRIES}), retrying in ${delay}ms`,
);
await this.sleep(delay);
}
}
if (lastError) {
this.logger.error(`OrdersCrone failed after ${MAX_DEADLOCK_RETRIES} retries`, lastError.stack);
throw lastError;
}
}
private sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
private async runCancelOldOnlinePendingOrders(): Promise<void> {
const cutoff = new Date(Date.now() - 15 * 60 * 1000);
this.logger.debug('Searching for pending online payments older than 15 minutes');
// Minimal query: only payment ids, no joins reduces lock contention and deadlock risk
const payments = await this.em.find(
Payment,
{
method: PaymentMethodEnum.Online,
status: PaymentStatusEnum.Pending,
createdAt: { $lte: cutoff },
},
{ fields: ['id'] },
);
if (!payments || payments.length === 0) {
return;
}
this.logger.log(`Found ${payments.length} stale pending online payments`);
for (const p of payments) {
try {
await this.em.transactional(async em => {
// reload inside transaction to avoid concurrency issues
const payment = await em.findOne(
Payment,
{ id: p.id },
{ populate: ['order', 'order.items', 'order.items.variant'] },
);
if (!payment) return;
if (payment.status !== PaymentStatusEnum.Pending) return;
payment.status = PaymentStatusEnum.Failed;
payment.failedAt = new Date();
if (payment.order) {
payment.order.status = OrderStatus.CANCELED;
}
em.persist(payment);
if (payment.order) em.persist(payment.order);
await em.flush();
this.logger.log(
`Marked payment ${payment.id} and order ${payment.order?.id} as failed`,
);
});
} catch (err) {
this.logger.error(`Error processing payment ${p.id}: ${(err as Error).message}`, (err as Error).stack);
}
} catch (err) {
this.logger.error(`OrdersCrone failed: ${err.message}`, err.stack);
}
}