To achieve high availability and better performance, we could build a HAProxy load balancer in front of multiple Ethereum RPC providers, and also automatically adjust traffic weights based on the latency and block timestamp of each RPC endpoints.
Configurations
In haproxy.cfg, we have a backend named rpc-backend, and two RPC endpoints: quicknode and alchemy as upstream servers.
global
log stdout format raw local0 info
stats socket ipv4@*:9999 level admin expose-fd listeners
stats timeout 5s
defaults
log global
mode http
option httplog
option dontlognull
timeout connect 10s
timeout client 60s
timeout server 60s
timeout http-request 60s
frontend stats
bind *:8404
stats enable
stats uri /
stats refresh 10s
frontend http
bind *:8000
option forwardfor
default_backend rpc-backend
backend rpc-backend
balance leastconn
server quicknode 127.0.0.1:8001 weight 100
server alchemy 127.0.0.1:8002 weight 100
frontend quicknode-frontend
bind *:8001
option dontlog-normal
default_backend quicknode-backend
backend quicknode-backend
balance roundrobin
http-request set-header Host xxx.quiknode.pro
http-request set-path /xxx
server quicknode xxx.quiknode.pro:443 sni str(xxx.quiknode.pro) check-ssl ssl verify none
frontend alchemy-frontend
bind *:8002
option dontlog-normal
default_backend alchemy-backend
backend alchemy-backend
balance roundrobin
http-request set-header Host xxx.alchemy.com
http-request set-path /xxx
server alchemy xxx.alchemy.com:443 sni str(xxx.alchemy.com) check-ssl ssl verify noneref:
https://docs.haproxy.org/2.7/configuration.html
https://www.haproxy.com/documentation/hapee/latest/configuration/
Test it on local:
docker run --rm -v $PWD:/usr/local/etc/haproxy \
-p 8000:8000 \
-p 8404:8404 \
-p 9999:9999 \
-i -t --name haproxy haproxy:2.7.0
docker exec -i -t -u 0 haproxy bash
echo "show stat" | socat stdio TCP:127.0.0.1:9999
echo "set weight rpc-backend/quicknode 0" | socat stdio TCP:127.0.0.1:9999
# if you're using a socket file descriptor
apt update
apt install socat -y
echo "set weight rpc-backend/alchemy 0" | socat stdio /var/lib/haproxy/haproxy.sockref:
https://www.redhat.com/sysadmin/getting-started-socat
Healtchcheck
Then the important part: we're going to run a simple but flexible healthcheck script, called node weighter, as a sidecar container. So the healthcheck script can access HAProxy admin socket of the HAProxy container through 127.0.0.1:9999.
The node weighter can be written in any language. Here is a TypeScript example:
in HAProxyConnector.ts which sets weights through HAProxy admin socket:
import net from "net"
export interface ServerWeight {
backendName: string
serverName: string
weight: number
}
export class HAProxyConnector {
constructor(readonly adminHost = "127.0.0.1", readonly adminPort = 9999) {}
setWeights(serverWeights: ServerWeight[]) {
const scaledServerWeights = this.scaleWeights(serverWeights)
const commands = scaledServerWeights.map(server => {
return `set weight ${server.backendName}/${server.serverName} ${server.weight}\n`
})
const client = net.createConnection({ host: this.adminHost, port: this.adminPort }, () => {
console.log("HAProxyAdminSocketConnected")
})
client.on("error", err => {
console.log("HAProxyAdminSocketError")
})
client.on("data", data => {
console.log("HAProxyAdminSocketData")
console.log(data.toString().trim())
})
client.write(commands.join(""))
}
private scaleWeights(serverWeights: ServerWeight[]) {
const totalWeight = sum(serverWeights.map(server => server.weight))
return serverWeights.map(server => {
server.weight = Math.floor((server.weight / totalWeight) * 256)
return server
})
}
}in RPCProxyWeighter.ts which calculates weights based a custom healthcheck logic:
import { HAProxyConnector } from "./connectors/HAProxyConnector"
import config from "./config.json"
export interface Server {
backendName: string
serverName: string
serverUrl: string
}
export interface ServerWithWeight {
backendName: string
serverName: string
weight: number
[metadata: string]: any
}
export class RPCProxyWeighter {
protected readonly log = Log.getLogger(RPCProxyWeighter.name)
protected readonly connector: HAProxyConnector
protected readonly ADJUST_INTERVAL_SEC = 60 // 60 seconds
protected readonly MAX_BLOCK_TIMESTAMP_DELAY_MSEC = 150 * 1000 // 150 seconds
protected readonly MAX_LATENCY_MSEC = 3 * 1000 // 3 seconds
protected shouldScale = false
protected totalWeight = 0
constructor() {
this.connector = new HAProxyConnector(config.admin.host, config.admin.port)
}
async start() {
while (true) {
let serverWithWeights = await this.calculateWeights(config.servers)
if (this.shouldScale) {
serverWithWeights = this.connector.scaleWeights(serverWithWeights)
}
this.connector.setWeights(serverWithWeights)
await sleep(1000 * this.ADJUST_INTERVAL_SEC)
}
}
async calculateWeights(servers: Server[]) {
this.totalWeight = 0
const serverWithWeights = await Promise.all(
servers.map(async server => {
try {
return await this.calculateWeight(server)
} catch (err: any) {
return {
backendName: server.backendName,
serverName: server.serverName,
weight: 0,
}
}
}),
)
// if all endpoints are unhealthy, overwrite weights to 100
if (this.totalWeight === 0) {
for (const server of serverWithWeights) {
server.weight = 100
}
}
return serverWithWeights
}
async calculateWeight(server: Server) {
const healthInfo = await this.getHealthInfo(server.serverUrl)
const serverWithWeight: ServerWithWeight = {
...{
backendName: server.backendName,
serverName: server.serverName,
weight: 0,
},
...healthInfo,
}
if (healthInfo.isBlockTooOld || healthInfo.isLatencyTooHigh) {
return serverWithWeight
}
// normalizedLatency: the lower the better
// blockTimestampDelayMsec: the lower the better
// both units are milliseconds at the same scale
// serverWithWeight.weight = 1 / healthInfo.normalizedLatency + 1 / healthInfo.blockTimestampDelayMsec
// NOTE: if we're using `balance source` in HAProxy, the weight can only be 100% or 0%,
// therefore, as long as the RPC endpoint is healthy, we always set the same weight
serverWithWeight.weight = 100
this.totalWeight += serverWithWeight.weight
return serverWithWeight
}
protected async getHealthInfo(serverUrl: string): Promise<HealthInfo> {
const provider = new ethers.providers.StaticJsonRpcProvider(serverUrl)
// TODO: add timeout
const start = Date.now()
const blockNumber = await provider.getBlockNumber()
const end = Date.now()
const block = await provider.getBlock(blockNumber)
const blockTimestamp = block.timestamp
const blockTimestampDelayMsec = Math.floor(Date.now() / 1000 - blockTimestamp) * 1000
const isBlockTooOld = blockTimestampDelayMsec >= this.MAX_BLOCK_TIMESTAMP_DELAY_MSEC
const latency = end - start
const normalizedLatency = this.normalizeLatency(latency)
const isLatencyTooHigh = latency >= this.MAX_LATENCY_MSEC
return {
blockNumber,
blockTimestamp,
blockTimestampDelayMsec,
isBlockTooOld,
latency,
normalizedLatency,
isLatencyTooHigh,
}
}
protected normalizeLatency(latency: number) {
if (latency <= 40) {
return 1
}
const digits = Math.floor(latency).toString().length
const base = Math.pow(10, digits - 1)
return Math.floor(latency / base) * base
}
}in config.json:
Technically, we don't need this config file. Instead, we could read the actual URLs from HAProxy admin socket directly. Though creating a JSON file that contains URLs is much simpler.
{
"admin": {
"host": "127.0.0.1",
"port": 9999
},
"servers": [
{
"backendName": "rpc-backend",
"serverName": "quicknode",
"serverUrl": "https://xxx.quiknode.pro/xxx"
},
{
"backendName": "rpc-backend",
"serverName": "alchemy",
"serverUrl": "https://xxx.alchemy.com/xxx"
}
]
}ref:
https://www.haproxy.com/documentation/hapee/latest/api/runtime-api/set-weight/
https://sleeplessbeastie.eu/2020/01/29/how-to-use-haproxy-stats-socket/
Deployments
apiVersion: v1
kind: ConfigMap
metadata:
name: rpc-proxy-config-file
data:
haproxy.cfg: |
...
config.json: |
...
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: rpc-proxy
spec:
replicas: 2
selector:
matchLabels:
app: rpc-proxy
template:
metadata:
labels:
app: rpc-proxy
spec:
volumes:
- name: rpc-proxy-config-file
configMap:
name: rpc-proxy-config-file
containers:
- name: haproxy
image: haproxy:2.7.0
ports:
- containerPort: 8000
protocol: TCP
resources:
requests:
cpu: 200m
memory: 256Mi
limits:
cpu: 1000m
memory: 256Mi
volumeMounts:
- name: rpc-proxy-config-file
subPath: haproxy.cfg
mountPath: /usr/local/etc/haproxy/haproxy.cfg
readOnly: true
- name: node-weighter
image: your-node-weighter
command: ["node", "./index.js"]
resources:
requests:
cpu: 200m
memory: 256Mi
limits:
cpu: 1000m
memory: 256Mi
volumeMounts:
- name: rpc-proxy-config-file
subPath: config.json
mountPath: /path/to/build/config.json
readOnly: true
---
apiVersion: v1
kind: Service
metadata:
name: rpc-proxy
spec:
clusterIP: None
selector:
app: rpc-proxy
ports:
- name: http
port: 8000
targetPort: 8000The RPC load balancer can then be accessed through http://rpc-proxy.default.svc.cluster.local:8000 inside the Kubernetes cluster.
ref:
https://www.containiq.com/post/kubernetes-sidecar-container
https://hub.docker.com/_/haproxy