// deployment/checklists/pre-deployment.js

const preDeploymentChecklist = {
codeReview: {
title: 'Code Review',
items: [
'□ All code reviewed by second developer',
'□ No hardcoded secrets in codebase',
'□ All TODO/FIXME items addressed',
'□ Console.log statements removed or converted to proper logging',
'□ Error handling covers all XRPL result codes',
'□ delivered_amount used (not Amount) for payment processing'
]
},

testing: {
title: 'Testing Complete',
items: [
'□ All unit tests passing',
'□ Integration tests on testnet passing',
'□ Load testing completed (if applicable)',
'□ Security testing completed',
'□ Edge cases tested (min/max values, concurrent requests)'
]
},

configuration: {
title: 'Configuration',
items: [
'□ Mainnet URLs configured (not testnet)',
'□ Production secrets in secret manager (not env files)',
'□ Appropriate rate limits set',
'□ Transaction limits configured conservatively',
'□ Logging level set appropriately (not debug)'
]
},

infrastructure: {
title: 'Infrastructure Ready',
items: [
'□ Production servers provisioned',
'□ TLS/HTTPS configured',
'□ Firewall rules in place',
'□ Database backups configured',
'□ Monitoring and alerting set up'
]
},

operations: {
title: 'Operations Prepared',
items: [
'□ Runbooks documented',
'□ On-call rotation established',
'□ Rollback procedure tested',
'□ Incident response plan ready',
'□ Communication channels defined'
]
},

wallets: {
title: 'Wallet Setup',
items: [
'□ Hot wallet funded with operational amount only',
'□ Warm/cold wallet structure in place',
'□ Master keys in secure cold storage',
'□ Regular keys configured for operations',
'□ Multi-sig set up for high-value operations'
]
}
};

module.exports = preDeploymentChecklist;
```

// config/production.js

const productionConfig = {
xrpl: {
// Mainnet servers - use multiple for redundancy
servers: [
'wss://xrplcluster.com',
'wss://s1.ripple.com',
'wss://s2.ripple.com'
],

// Longer timeouts for production reliability
connectionTimeout: 30000,
requestTimeout: 30000,

// Connection pool
poolSize: 5,

// Health check interval
healthCheckInterval: 30000
},

limits: {
// Start conservative - increase after validation
maxPaymentXRP: 100, // Low initial limit
dailyLimitXRP: 1000, // Low initial limit
minPaymentXRP: 0.001,
maxPendingPayments: 50
},

security: {
// Rate limiting
rateLimitPerMinute: 30,
rateLimitPerHour: 500,

// IP restrictions (if applicable)
allowedIPs: process.env.ALLOWED_IPS?.split(',') || [],

// Request validation
requireAuthentication: true,
maxRequestBodySize: '10kb'
},

monitoring: {
// Metrics
metricsEnabled: true,
metricsPort: 9090,

// Logging
logLevel: 'info', // Not 'debug' in production

// Alerts
alertOnError: true,
alertOnHighLatency: true,
latencyThresholdMs: 5000
},

// Graceful shutdown
shutdownTimeout: 30000
};

// Validate production config
function validateProductionConfig(config) {
const errors = [];

if (config.xrpl.servers.some(s => s.includes('testnet'))) {
errors.push('Testnet URLs found in production config');
}

if (config.limits.maxPaymentXRP > 10000) {
console.warn('Warning: High max payment limit. Ensure this is intended.');
}

if (!config.security.requireAuthentication) {
errors.push('Authentication should be required in production');
}

if (errors.length > 0) {
throw new Error(Production config validation failed: ${errors.join(', ')});
}
}

validateProductionConfig(productionConfig);

module.exports = productionConfig;
```

// scripts/setup-mainnet-wallet.js
const xrpl = require('xrpl');
const readline = require('readline');

async function setupMainnetWallet() {
console.log('=== Mainnet Wallet Setup ===\n');
console.log('WARNING: This creates real wallets for real XRP.\n');

const rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});

const confirm = await question(rl, 'Type "I UNDERSTAND" to continue: ');
if (confirm !== 'I UNDERSTAND') {
console.log('Aborted.');
process.exit(1);
}

// Option 1: Generate new wallet
console.log('\n--- Option 1: Generate New Wallet ---');
const newWallet = xrpl.Wallet.generate();
console.log(Address: ${newWallet.address});
console.log(Seed: ${newWallet.seed});
console.log('\n⚠️ SAVE THIS SEED SECURELY. It cannot be recovered.\n');

// Option 2: Import existing wallet
console.log('--- Option 2: Import Existing ---');
console.log('To import, use: xrpl.Wallet.fromSeed("your_seed")');

// Setup recommendations
console.log('\n=== Setup Recommendations ===');
console.log('1. Fund hot wallet with minimum operational amount');
console.log('2. Set up regular key for daily operations');
console.log('3. Store master seed in cold storage (offline)');
console.log('4. Configure multi-sig for large transactions');
console.log('5. Enable RequireDest if using destination tags');

rl.close();
}

function question(rl, prompt) {
return new Promise(resolve => rl.question(prompt, resolve));
}

setupMainnetWallet().catch(console.error);
```

---

# docker-compose.production.yml

version: '3.8'

services:
app:
build: .
restart: always
environment:
- NODE_ENV=production
- XRPL_SERVERS=wss://xrplcluster.com,wss://s1.ripple.com
secrets:
- hot_wallet_seed
deploy:
replicas: 2
resources:
limits:
cpus: '1'
memory: 1G
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:3000/health"]" target="_blank" rel="noopener noreferrer" class="text-cyan-400 hover:text-cyan-300 underline hover:no-underline transition-colors inline-flex items-center gap-1">http://localhost:3000/health%22%5D">http://localhost:3000/health"]
interval: 30s
timeout: 10s
retries: 3
networks:
- app-network
- monitoring

nginx:
image: nginx:alpine
restart: always
ports:
- "443:443"
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf:ro
- ./certs:/etc/nginx/certs:ro
depends_on:
- app
networks:
- app-network

prometheus:
image: prom/prometheus
restart: always
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus_data:/prometheus
networks:
- monitoring

grafana:
image: grafana/grafana
restart: always
volumes:
- grafana_data:/var/lib/grafana
environment:
- GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD}
networks:
- monitoring

secrets:
hot_wallet_seed:
external: true

volumes:
prometheus_data:
grafana_data:

networks:
app-network:
monitoring:
```

# nginx.conf

upstream app_servers {
least_conn;
server app:3000;
}

server {
listen 443 ssl http2;
server_name api.example.com;

ssl_certificate /etc/nginx/certs/fullchain.pem;
ssl_certificate_key /etc/nginx/certs/privkey.pem;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256;

Security headers

add_header X-Frame-Options "SAMEORIGIN" always;
add_header X-Content-Type-Options "nosniff" always;
add_header X-XSS-Protection "1; mode=block" always;
add_header Strict-Transport-Security "max-age=31536000" always;

Rate limiting

limit_req_zone $binary_remote_addr zone=api:10m rate=10r/s;
limit_req zone=api burst=20 nodelay;

location / {
proxy_pass http://app_servers" target="_blank" rel="noopener noreferrer" class="text-cyan-400 hover:text-cyan-300 underline hover:no-underline transition-colors inline-flex items-center gap-1">http://app_servers">http://app_servers ;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_cache_bypass $http_upgrade;

Timeouts

    proxy_connect_timeout 30s;
    proxy_send_timeout 30s;
    proxy_read_timeout 30s;
}

location /health {
proxy_pass http://app_servers/health" target="_blank" rel="noopener noreferrer" class="text-cyan-400 hover:text-cyan-300 underline hover:no-underline transition-colors inline-flex items-center gap-1">http://app_servers/health">http://app_servers/health ;
access_log off;
}
}
```

// src/process-manager.js

class ProcessManager {
constructor(app) {
this.app = app;
this.isShuttingDown = false;
this.connections = new Set();
}

start(port) {
const server = this.app.listen(port, () => {
console.log(Server started on port ${port});
});

// Track connections for graceful shutdown
server.on('connection', (conn) => {
this.connections.add(conn);
conn.on('close', () => this.connections.delete(conn));
});

// Graceful shutdown handlers
process.on('SIGTERM', () => this.shutdown(server, 'SIGTERM'));
process.on('SIGINT', () => this.shutdown(server, 'SIGINT'));

// Uncaught error handlers
process.on('uncaughtException', (error) => {
console.error('Uncaught exception:', error);
this.shutdown(server, 'uncaughtException');
});

process.on('unhandledRejection', (reason, promise) => {
console.error('Unhandled rejection:', reason);
// Don't shutdown - log and monitor
});

return server;
}

async shutdown(server, signal) {
if (this.isShuttingDown) return;
this.isShuttingDown = true;

console.log(\nReceived ${signal}, starting graceful shutdown...);

// Stop accepting new connections
server.close(() => {
console.log('HTTP server closed');
});

// Close existing connections
for (const conn of this.connections) {
conn.end();
}

// Wait for in-flight requests (with timeout)
const shutdownTimeout = 30000;
const timeoutPromise = new Promise((_, reject) => {
setTimeout(() => reject(new Error('Shutdown timeout')), shutdownTimeout);
});

try {
await Promise.race([
this.cleanupResources(),
timeoutPromise
]);
console.log('Graceful shutdown complete');
process.exit(0);
} catch (error) {
console.error('Shutdown error:', error);
process.exit(1);
}
}

async cleanupResources() {
// Close XRPL connections
// Close database connections
// Flush metrics
// etc.
}
}

module.exports = ProcessManager;
```

---

// src/monitoring/metrics.js
const promClient = require('prom-client');

// Create metrics
const metrics = {
// Request metrics
httpRequestDuration: new promClient.Histogram({
name: 'http_request_duration_seconds',
help: 'HTTP request duration in seconds',
labelNames: ['method', 'route', 'status'],
buckets: [0.1, 0.5, 1, 2, 5]
}),

httpRequestTotal: new promClient.Counter({
name: 'http_requests_total',
help: 'Total HTTP requests',
labelNames: ['method', 'route', 'status']
}),

// XRPL metrics
xrplRequestDuration: new promClient.Histogram({
name: 'xrpl_request_duration_seconds',
help: 'XRPL request duration in seconds',
labelNames: ['command'],
buckets: [0.1, 0.5, 1, 2, 5, 10]
}),

xrplConnectionStatus: new promClient.Gauge({
name: 'xrpl_connection_status',
help: 'XRPL connection status (1=connected, 0=disconnected)',
labelNames: ['server']
}),

// Business metrics
paymentsTotal: new promClient.Counter({
name: 'payments_total',
help: 'Total payments processed',
labelNames: ['status']
}),

paymentAmountTotal: new promClient.Counter({
name: 'payment_amount_xrp_total',
help: 'Total XRP amount processed'
}),

walletBalance: new promClient.Gauge({
name: 'wallet_balance_xrp',
help: 'Current wallet balance in XRP',
labelNames: ['wallet_type']
}),

pendingPayments: new promClient.Gauge({
name: 'pending_payments',
help: 'Number of pending payments'
})
};

// Middleware for HTTP metrics
function httpMetricsMiddleware(req, res, next) {
const start = Date.now();

res.on('finish', () => {
const duration = (Date.now() - start) / 1000;
const route = req.route?.path || req.path;

metrics.httpRequestDuration
.labels(req.method, route, res.statusCode)
.observe(duration);

metrics.httpRequestTotal
.labels(req.method, route, res.statusCode)
.inc();
});

next();
}

// Expose metrics endpoint
function metricsHandler(req, res) {
res.set('Content-Type', promClient.register.contentType);
promClient.register.metrics().then(data => res.send(data));
}

module.exports = { metrics, httpMetricsMiddleware, metricsHandler };
```

# prometheus-alerts.yml

groups:

• name: xrpl-application
  rules:

  High error rate

  • alert: HighErrorRate
    expr: rate(http_requests_total{status=~"5.."}[5m]) / rate(http_requests_total[5m]) > 0.05
    for: 5m
    labels:
    severity: critical
    annotations:
    summary: High error rate detected
    description: "Error rate is {{ $value | printf "%.2f" }}% over the last 5 minutes"

XRPL connection down

  - alert: XRPLConnectionDown
    expr: xrpl_connection_status == 0
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: XRPL connection lost
      description: "Connection to {{ $labels.server }} has been down for 1 minute"

Low wallet balance

  - alert: LowWalletBalance
    expr: wallet_balance_xrp{wallet_type="hot"} < 100
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: Hot wallet balance low
      description: "Hot wallet balance is {{ $value }} XRP"

Critical wallet balance

  - alert: CriticalWalletBalance
    expr: wallet_balance_xrp{wallet_type="hot"} < 20
    for: 1m
    labels:
      severity: critical
    annotations:
      summary: Hot wallet critically low
      description: "Hot wallet balance is {{ $value }} XRP - immediate action required"

Payment failures

  - alert: PaymentFailureSpike
    expr: rate(payments_total{status="failed"}[5m]) > 0.1
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: Elevated payment failures
      description: "Payment failure rate is {{ $value }} per second"

High latency

  - alert: HighLatency
    expr: histogram_quantile(0.95, rate(xrpl_request_duration_seconds_bucket[5m])) > 5
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: High XRPL request latency
      description: "95th percentile latency is {{ $value }} seconds"

Pending payments stuck

  - alert: PendingPaymentsStuck
    expr: pending_payments > 10 and delta(pending_payments[10m]) >= 0
    for: 10m
    labels:
      severity: warning
    annotations:
      summary: Payments not confirming
      description: "{{ $value }} payments pending with no decrease in 10 minutes"

// src/api/health.js

async function healthCheck(req, res) {
const checks = {
timestamp: new Date().toISOString(),
status: 'healthy',
checks: {}
};

let allHealthy = true;

// Check XRPL connection
try {
const start = Date.now();
await xrplService.request({ command: 'server_info' });
checks.checks.xrpl = {
status: 'healthy',
latencyMs: Date.now() - start
};
} catch (error) {
checks.checks.xrpl = {
status: 'unhealthy',
error: error.message
};
allHealthy = false;
}

// Check wallet balance
try {
const balance = await paymentService.getBalance();
const isHealthy = balance.available > 10; // Minimum 10 XRP
checks.checks.wallet = {
status: isHealthy ? 'healthy' : 'degraded',
balance: balance.available
};
if (!isHealthy) allHealthy = false;
} catch (error) {
checks.checks.wallet = {
status: 'unhealthy',
error: error.message
};
allHealthy = false;
}

// Check pending payments
const pending = paymentService.getPendingPayments();
const oldestPending = pending.reduce((oldest, p) => {
const age = Date.now() - p.submittedAt.getTime();
return age > oldest ? age : oldest;
}, 0);

checks.checks.payments = {
status: oldestPending < 300000 ? 'healthy' : 'degraded', // 5 min threshold
pending: pending.length,
oldestAgeSeconds: Math.floor(oldestPending / 1000)
};

// Overall status
checks.status = allHealthy ? 'healthy' : 'unhealthy';

const statusCode = allHealthy ? 200 : 503;
res.status(statusCode).json(checks);
}

module.exports = healthCheck;
```

---

# Runbook: Hot Wallet Refill

Not applicable - transfers are irreversible.

If warm wallet also low, escalate to cold wallet custodians.
```

# Runbook: XRPL Connection Issues

Check logs

kubectl logs -l app=payment-service --tail=100
```

# Runbook: Payment Not Confirming

---

// deployment/staged-rollout.js

const rolloutStages = {
stage1: {
name: 'Internal Testing',
duration: '24 hours',
criteria: {
maxPayment: 10, // XRP
dailyLimit: 100,
allowedUsers: ['internal_test_account']
},
successCriteria: [
'No errors in 24 hours',
'All test payments confirmed',
'Monitoring working correctly'
]
},

stage2: {
name: 'Limited Beta',
duration: '1 week',
criteria: {
maxPayment: 100,
dailyLimit: 1000,
allowedUsers: 'beta_list'
},
successCriteria: [
'No critical errors',
'Error rate < 1%',
'All payments confirmed within 2 minutes',
'No security incidents'
]
},

stage3: {
name: 'Public Beta',
duration: '2 weeks',
criteria: {
maxPayment: 500,
dailyLimit: 5000,
allowedUsers: 'all'
},
successCriteria: [
'Stable performance under load',
'Error rate < 0.1%',
'Customer feedback positive'
]
},

stage4: {
name: 'General Availability',
criteria: {
maxPayment: 'configured_limit',
dailyLimit: 'configured_limit',
allowedUsers: 'all'
}
}
};

function evaluateStage(stage, metrics) {
const results = {
stage: stage.name,
passed: true,
failures: []
};

// Evaluate each success criterion
for (const criterion of stage.successCriteria || []) {
const passed = evaluateCriterion(criterion, metrics);
if (!passed) {
results.passed = false;
results.failures.push(criterion);
}
}

return results;
}
```

# Launch Day Checklist

---

Production deployment is as much about operations as code. The best code fails without proper infrastructure, monitoring, and procedures. Invest in operational readiness—it determines whether your application survives its first incident.

---

Assignment: Create a complete production deployment package.

Requirements:

• Production configuration file

• Environment variable documentation

• Secret management setup

• Docker/container configuration

• Reverse proxy setup (NGINX)

• Health check endpoints

• Metrics collection

• Alert rules

• Dashboard configuration

• Pre-deployment checklist

• At least 3 runbooks

• Staged rollout plan

• Configuration complete and validated (25%)

• Infrastructure production-ready (25%)

• Monitoring comprehensive (25%)

• Operations documented (25%)

Time investment: 4-5 hours
Value: Actual production deployment artifacts

---

For Next Lesson:
You've deployed to production. Lesson 20 covers what's next—your continued learning path, community resources, and advanced topics to explore.

---

End of Lesson 19

Total words: ~5,000
Estimated completion time: 55 minutes reading + 4-5 hours for deliverable