diff --git a/cli/config.go b/cli/config.go new file mode 100644 index 00000000..7df802fc --- /dev/null +++ b/cli/config.go @@ -0,0 +1,104 @@ +package cli + +import ( + "os" + "strings" + + "github.com/flashbots/mev-boost/server/types" + "gopkg.in/yaml.v3" +) + +type RelayConfigYAML struct { + URL string `yaml:"url"` + ID string `yaml:"id"` + EnableTimingGames bool `yaml:"enable_timing_games"` + TargetFirstRequestMs uint64 `yaml:"target_first_request_ms"` + FrequencyGetHeaderMs uint64 `yaml:"frequency_getheader_ms"` +} + +// Config holds all configuration settings from the config file +type Config struct { + TimeoutGetHeaderMs uint64 `yaml:"timeout_get_header_ms"` + LateInSlotTimeMs uint64 `yaml:"late_in_slot_time_ms"` + Relays []RelayConfigYAML `yaml:"relays"` +} + +type ConfigResult struct { + RelayConfigs map[string]types.RelayConfig + TimeoutGetHeaderMs uint64 + LateInSlotTimeMs uint64 +} + +// LoadConfigFile loads configurations from a YAML file +func LoadConfigFile(configPath string) (*ConfigResult, error) { + data, err := os.ReadFile(configPath) + if err != nil { + return nil, err + } + + var config Config + if err := yaml.Unmarshal(data, &config); err != nil { + return nil, err + } + + timeoutGetHeaderMs := config.TimeoutGetHeaderMs + if timeoutGetHeaderMs == 0 { + timeoutGetHeaderMs = 900 + } + + lateInSlotTimeMs := config.LateInSlotTimeMs + if lateInSlotTimeMs == 0 { + lateInSlotTimeMs = 1000 + } + + configMap := make(map[string]types.RelayConfig) + for _, relay := range config.Relays { + relayEntry, err := types.NewRelayEntry(strings.TrimSpace(relay.URL)) + if err != nil { + return nil, err + } + if relay.ID != "" { + relayEntry.ID = relay.ID + } else { + relayEntry.ID = relayEntry.URL.String() + } + relayConfig := types.RelayConfig{ + RelayEntry: relayEntry, + EnableTimingGames: relay.EnableTimingGames, + TargetFirstRequestMs: relay.TargetFirstRequestMs, + FrequencyGetHeaderMs: relay.FrequencyGetHeaderMs, + } + configMap[relayEntry.String()] = relayConfig + } + + return &ConfigResult{ + RelayConfigs: configMap, + TimeoutGetHeaderMs: timeoutGetHeaderMs, + LateInSlotTimeMs: lateInSlotTimeMs, + }, nil +} + +// MergeRelayConfigs merges relays passed via --relays with config file settings. +// this allows the users to still use --relays if they dont want to provide a config file +func MergeRelayConfigs(relays []types.RelayEntry, configMap map[string]types.RelayConfig) []types.RelayConfig { + configs := make([]types.RelayConfig, 0) + processedURLs := make(map[string]bool) + + for _, entry := range relays { + urlStr := entry.String() + if config, exists := configMap[urlStr]; exists { + config.RelayEntry = entry + configs = append(configs, config) + } else { + configs = append(configs, types.NewRelayConfig(entry)) + } + processedURLs[urlStr] = true + } + + for urlStr, config := range configMap { + if !processedURLs[urlStr] { + configs = append(configs, config) + } + } + return configs +} diff --git a/cli/flags.go b/cli/flags.go index f147c12b..47c9ae6a 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -29,6 +29,7 @@ var flags = []cli.Flag{ hoodiFlag, // relay relaysFlag, + relayConfigFlag, deprecatedRelayMonitorFlag, minBidFlag, relayCheckFlag, @@ -135,6 +136,12 @@ var ( Usage: "relay urls - single entry or comma-separated list (scheme://pubkey@host)", Category: RelayCategory, } + relayConfigFlag = &cli.StringFlag{ + Name: "config", + Sources: cli.EnvVars("CONFIG_FILE"), + Usage: "path to YAML configuration file", + Category: RelayCategory, + } deprecatedRelayMonitorFlag = &cli.StringSliceFlag{ Name: "relay-monitors", Aliases: []string{"relay-monitor"}, diff --git a/cli/main.go b/cli/main.go index b7f7ef60..1ff7743c 100644 --- a/cli/main.go +++ b/cli/main.go @@ -13,6 +13,7 @@ import ( "github.com/flashbots/mev-boost/common" "github.com/flashbots/mev-boost/config" "github.com/flashbots/mev-boost/server" + serverTypes "github.com/flashbots/mev-boost/server/types" "github.com/sirupsen/logrus" "github.com/urfave/cli/v3" ) @@ -29,6 +30,14 @@ const ( genesisTimeHoodi = 1742213400 ) +type RelaySetupResult struct { + RelayConfigs []serverTypes.RelayConfig + MinBid types.U256Str + RelayCheck bool + TimeoutGetHeaderMs uint64 + LateInSlotTimeMs uint64 +} + var ( // errors errInvalidLoglevel = errors.New("invalid loglevel") @@ -66,7 +75,7 @@ func start(_ context.Context, cmd *cli.Command) error { var ( genesisForkVersion, genesisTime = setupGenesis(cmd) - relays, minBid, relayCheck = setupRelays(cmd) + relaySetup = setupRelays(cmd) listenAddr = cmd.String(addrFlag.Name) metricsEnabled = cmd.Bool(metricsFlag.Name) metricsAddr = cmd.String(metricsAddrFlag.Name) @@ -75,23 +84,25 @@ func start(_ context.Context, cmd *cli.Command) error { opts := server.BoostServiceOpts{ Log: log, ListenAddr: listenAddr, - Relays: relays, + RelayConfigs: relaySetup.RelayConfigs, GenesisForkVersionHex: genesisForkVersion, GenesisTime: genesisTime, - RelayCheck: relayCheck, - RelayMinBid: minBid, + RelayCheck: relaySetup.RelayCheck, + RelayMinBid: relaySetup.MinBid, RequestTimeoutGetHeader: time.Duration(cmd.Int(timeoutGetHeaderFlag.Name)) * time.Millisecond, RequestTimeoutGetPayload: time.Duration(cmd.Int(timeoutGetPayloadFlag.Name)) * time.Millisecond, RequestTimeoutRegVal: time.Duration(cmd.Int(timeoutRegValFlag.Name)) * time.Millisecond, RequestMaxRetries: cmd.Int(maxRetriesFlag.Name), MetricsAddr: metricsAddr, + TimeoutGetHeaderMs: relaySetup.TimeoutGetHeaderMs, + LateInSlotTimeMs: relaySetup.LateInSlotTimeMs, } service, err := server.NewBoostService(opts) if err != nil { log.WithError(err).Fatal("failed creating the server") } - if relayCheck && service.CheckRelays() == 0 { + if relaySetup.RelayCheck && service.CheckRelays() == 0 { log.Error("no relay passed the health-check!") } @@ -108,7 +119,7 @@ func start(_ context.Context, cmd *cli.Command) error { return service.StartHTTPServer() } -func setupRelays(cmd *cli.Command) (relayList, types.U256Str, bool) { +func setupRelays(cmd *cli.Command) RelaySetupResult { // For backwards compatibility with the -relays flag. var relays relayList if cmd.IsSet(relaysFlag.Name) { @@ -125,9 +136,32 @@ func setupRelays(cmd *cli.Command) (relayList, types.U256Str, bool) { if len(relays) == 0 { log.Fatal("no relays specified") } - log.Infof("using %d relays", len(relays)) - for index, relay := range relays { - log.Infof("relay #%d: %s", index+1, relay.String()) + + // load configuration via config file + var configMap map[string]serverTypes.RelayConfig + var timeoutGetHeaderMs uint64 = 900 + var lateInSlotTimeMs uint64 = 1000 + if cmd.IsSet(relayConfigFlag.Name) { + configPath := cmd.String(relayConfigFlag.Name) + log.Infof("loading config from: %s", configPath) + configResult, err := LoadConfigFile(configPath) + if err != nil { + log.WithError(err).Fatal("failed to load config file") + } else { + configMap = configResult.RelayConfigs + timeoutGetHeaderMs = configResult.TimeoutGetHeaderMs + lateInSlotTimeMs = configResult.LateInSlotTimeMs + } + } + relayConfigs := MergeRelayConfigs(relays, configMap) + + log.Infof("using %d relays", len(relayConfigs)) + for index, config := range relayConfigs { + if config.EnableTimingGames { + log.Infof("relay #%d: %s timing games: enabled", index+1, config.RelayEntry.String()) + } else { + log.Infof("relay #%d: %s", index+1, config.RelayEntry.String()) + } } relayMinBidWei, err := sanitizeMinBid(cmd.Float(minBidFlag.Name)) @@ -137,7 +171,13 @@ func setupRelays(cmd *cli.Command) (relayList, types.U256Str, bool) { if relayMinBidWei.BigInt().Sign() > 0 { log.Infof("min bid set to %v eth (%v wei)", cmd.Float(minBidFlag.Name), relayMinBidWei) } - return relays, *relayMinBidWei, cmd.Bool(relayCheckFlag.Name) + return RelaySetupResult{ + RelayConfigs: relayConfigs, + MinBid: *relayMinBidWei, + RelayCheck: cmd.Bool(relayCheckFlag.Name), + TimeoutGetHeaderMs: timeoutGetHeaderMs, + LateInSlotTimeMs: lateInSlotTimeMs, + } } func setupGenesis(cmd *cli.Command) (string, uint64) { diff --git a/examples/config.yaml b/examples/config.yaml new file mode 100644 index 00000000..19d8a91e --- /dev/null +++ b/examples/config.yaml @@ -0,0 +1,24 @@ +# Example configuration for mev-boost + +# timeout settings for get_header requests +timeout_get_header_ms: 900 # timeout for get_header request in milliseconds +late_in_slot_time_ms: 1000 # threshold that defines when in a slot is considered "too late" + +# Relay configurations +relays: + # relay with timing games enabled + - url: https://0x9000009807ed12c1f08bf4e81c6da3ba8e3fc3d953898ce0102433094e5f22f21102ec057841fcb81978ed1ea0fa8246@relay.relayer1.net + # id for identifying the relay. if not provided, the url will be used as the id + id: relay.relayer1.net + enable_timing_games: true + # time in ms from slot start for the first getHeader request + target_first_request_ms: 200 + # time in ms between subsequent getHeader requests + frequency_getheader_ms: 100 + + # relay with timing games disabled (standard behavior) + - url: https://0x9000009807ed12c1f08bf4e81c6da3ba8e3fc3d953898ce0102433094e5f22f21102ec057841fcb81978ed1ea0fa8246@relay.relayer2.com + id: relay.relayer2.com + enable_timing_games: false + target_first_request_ms: 0 + frequency_getheader_ms: 0 diff --git a/go.mod b/go.mod index cdf8b56f..7571fc99 100644 --- a/go.mod +++ b/go.mod @@ -61,5 +61,5 @@ require ( golang.org/x/crypto v0.37.0 // indirect golang.org/x/sys v0.32.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/yaml.v3 v3.0.1 ) diff --git a/server/get_header.go b/server/get_header.go index c70d768f..c93b7549 100644 --- a/server/get_header.go +++ b/server/get_header.go @@ -25,8 +25,20 @@ import ( "github.com/sirupsen/logrus" ) +type relayBid struct { + bid *builderSpec.VersionedSignedBuilderBid + relay types.RelayEntry + contentType string +} + +type bidResult struct { + bid *builderSpec.VersionedSignedBuilderBid + contentType string + timestamp time.Time +} + // getHeader requests a bid from each relay and returns the most profitable one -func (m *BoostService) getHeader(log *logrus.Entry, slot phase0.Slot, pubkey, parentHashHex string, ua UserAgent, proposerAcceptContentTypes string) (bidResp, error) { +func (m *BoostService) getHeader(log *logrus.Entry, slot phase0.Slot, pubkey, parentHashHex string, ua UserAgent, proposerAcceptContentTypes string, userTimeout uint64) (bidResp, error) { // Ensure arguments are valid if len(pubkey) != 98 { return bidResp{}, errInvalidPubkey @@ -47,7 +59,6 @@ func (m *BoostService) getHeader(log *logrus.Entry, slot phase0.Slot, pubkey, pa // Compute these once, instead of for each relay userAgent := wrapUserAgent(ua) - startTime := fmt.Sprintf("%d", time.Now().UTC().UnixMilli()) // Log how late into the slot the request starts slotStartTimestamp := m.genesisTime + uint64(slot)*config.SlotTimeSec @@ -59,218 +70,375 @@ func (m *BoostService) getHeader(log *logrus.Entry, slot phase0.Slot, pubkey, pa }).Infof("getHeader request start - %d milliseconds into slot %d", msIntoSlot, slot) var ( - mu sync.Mutex - wg sync.WaitGroup + mu sync.Mutex + wg sync.WaitGroup + relayBids []relayBid + maxTimeoutMs uint64 + ) - // The final response, containing the highest bid (if any) - result = bidResp{} + if m.timeoutGetHeaderMs < m.lateInSlotTimeMs-msIntoSlot { + maxTimeoutMs = m.timeoutGetHeaderMs + } else { + maxTimeoutMs = m.lateInSlotTimeMs - msIntoSlot + } - // Relays that sent the bid for a specific blockHash - relays = make(map[BlockHashHex][]types.RelayEntry) - ) + if maxTimeoutMs == 0 { + return bidResp{}, nil + } + + if userTimeout > 0 { + if userTimeout < maxTimeoutMs { + maxTimeoutMs = userTimeout + } + } // Request a bid from each relay - for _, relay := range m.relays { + for _, relayConfig := range m.relayConfigs { wg.Add(1) - go func(relay types.RelayEntry) { + go func(relayConfig types.RelayConfig) { + relay := relayConfig.RelayEntry defer wg.Done() // Build the request URL url := relay.GetURI(fmt.Sprintf("/eth/v1/builder/header/%d/%s/%s", slot, parentHashHex, pubkey)) log := log.WithField("url", url) - // Make a new request - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) - if err != nil { - log.WithError(err).Warn("error creating new request") - return - } + var bid *builderSpec.VersionedSignedBuilderBid + var contentType string - // Add header fields to this request - req.Header.Set(HeaderAccept, proposerAcceptContentTypes) - req.Header.Set(HeaderKeySlotUID, slotUID.String()) - req.Header.Set(HeaderUserAgent, userAgent) - req.Header.Set(HeaderDateMilliseconds, startTime) - req.Header.Set(HeaderTimeoutMs, strconv.FormatInt(m.httpClientGetHeader.Timeout.Milliseconds(), 10)) - - // Send the request - log.Debug("requesting header") - start := time.Now() - resp, err := m.httpClientGetHeader.Do(req) - RecordRelayLatency(params.PathGetHeader, relay.String(), float64(time.Since(start).Microseconds())) - if err != nil { - log.WithError(err).Warn("error calling getHeader on relay") - return + // check if timing games enabled + if relayConfig.EnableTimingGames { + bid, contentType = m.handleTimingGamesGetHeader(log, relayConfig, url, slotUID, userAgent, proposerAcceptContentTypes, msIntoSlot, maxTimeoutMs) + } else { + bid, contentType = m.sendGetHeaderRequest(log, relay, url, slotUID, userAgent, proposerAcceptContentTypes) } - defer resp.Body.Close() - RecordRelayStatusCode(strconv.Itoa(resp.StatusCode), params.PathGetHeader, relay.String()) - // Check if no header is available - if resp.StatusCode == http.StatusNoContent { - log.Debug("no-content response") - return - } + if bid != nil { + bidInfo, err := parseBidInfo(bid) + if err == nil { + valueEth := weiBigIntToEthBigFloat(bidInfo.value.ToBig()) + valueEthFloat64, _ := valueEth.Float64() + RecordBidValue(relay.GetID(), valueEthFloat64) + } - // Check that the response was successful - if resp.StatusCode != http.StatusOK { - err = fmt.Errorf("%w: %d", errHTTPErrorResponse, resp.StatusCode) - log.WithError(err).Warn("error status code") - return + mu.Lock() + relayBids = append(relayBids, relayBid{bid: bid, relay: relay, contentType: contentType}) + mu.Unlock() } + }(relayConfig) + } + wg.Wait() - // Get the resp body content - respBytes, err := io.ReadAll(resp.Body) - if err != nil { - log.WithError(err).Warn("error reading response body") - return - } + var ( + result = bidResp{} + relays = make(map[BlockHashHex][]types.RelayEntry) + ) - // Get the response's content type, default to JSON - respContentType, _, err := mime.ParseMediaType(resp.Header.Get(HeaderContentType)) - if err != nil { - log.WithError(err).Warn("error parsing response content type") - respContentType = MediaTypeJSON - } - log = log.WithField("respContentType", respContentType) + // process the bids and select the one with the best value + for _, rb := range relayBids { + m.processBid(log, rb.relay, rb.bid, rb.contentType, parentHashHex, &result, relays, slot) + } - // Get the optional version, used with SSZ decoding - respEthConsensusVersion := resp.Header.Get(HeaderEthConsensusVersion) - log = log.WithField("respEthConsensusVersion", respEthConsensusVersion) + // Set the winning relays before returning + result.relays = relays[BlockHashHex(result.bidInfo.blockHash.String())] - // Decode bid - bid := new(builderSpec.VersionedSignedBuilderBid) - err = decodeBid(respBytes, respContentType, respEthConsensusVersion, bid) - if err != nil { - log.WithError(err).Warn("error decoding bid") - return - } + if len(result.relays) > 0 { + RecordWinningBidValue(result.bidInfo.value.Float64()) + } - // Skip if bid is empty - if bid.IsEmpty() { - log.Debug("skipping empty bid") - return - } + return result, nil +} - // Getting the bid info will check if there are missing fields in the response - bidInfo, err := parseBidInfo(bid) - if err != nil { - log.WithError(err).Warn("error parsing bid info") - return - } +// handleTimingGamesGetHeader implements timing games strategy for a relay +// Returns the latest bid and its content type from multiple timed requests +func (m *BoostService) handleTimingGamesGetHeader( + log *logrus.Entry, + relayConfig types.RelayConfig, + url string, + slotUID uuid.UUID, + userAgent string, + proposerAcceptContentTypes string, + msIntoSlot uint64, + timeoutLeftMs uint64, +) (*builderSpec.VersionedSignedBuilderBid, string) { + relay := relayConfig.RelayEntry + + // wait til target time is configured + if relayConfig.TargetFirstRequestMs > 0 { + delayMs := relayConfig.TargetFirstRequestMs - msIntoSlot + if delayMs > 0 { + log.WithFields(logrus.Fields{ + "targetMs": relayConfig.TargetFirstRequestMs, + "msIntoSlot": msIntoSlot, + "delayMs": delayMs, + }).Debug("waiting to send header request via timing games") + timeoutLeftMs -= delayMs + time.Sleep(time.Duration(delayMs) * time.Millisecond) + } + } - // Ignore bids with an empty block - if bidInfo.blockHash == nilHash { - log.Warn("relay responded with empty block hash") - return - } + // send multiple requests at frequency intervals + if relayConfig.FrequencyGetHeaderMs > 0 { //nolint:nestif + log.WithFields(logrus.Fields{ + "frequencyMs": relayConfig.FrequencyGetHeaderMs, + "timeoutLeftMs": timeoutLeftMs, + }).Debug("sending multiple header requests via timing games") + + var bidResults []bidResult + var mu sync.Mutex + + // keep sending requests until time runs out + var wg sync.WaitGroup + for timeoutLeftMs > 0 { + wg.Add(1) + go func() { + defer wg.Done() + bid, contentType := m.sendGetHeaderRequest(log, relay, url, slotUID, userAgent, proposerAcceptContentTypes) + if bid != nil { + mu.Lock() + bidResults = append(bidResults, bidResult{ + bid: bid, + contentType: contentType, + timestamp: time.Now(), + }) + mu.Unlock() + } + }() - // Add some info about the bid to the logger - valueEth := weiBigIntToEthBigFloat(bidInfo.value.ToBig()) - log = log.WithFields(logrus.Fields{ - "blockNumber": bidInfo.blockNumber, - "blockHash": bidInfo.blockHash.String(), - "txRoot": bidInfo.txRoot.String(), - "value": valueEth.Text('f', 18), - }) - - // Ensure the bid uses the correct public key - if relay.PublicKey.String() != bidInfo.pubkey.String() { - log.Errorf("bid pubkey mismatch. expected: %s - got: %s", relay.PublicKey.String(), bidInfo.pubkey.String()) - return + if timeoutLeftMs > relayConfig.FrequencyGetHeaderMs { + timeoutLeftMs -= relayConfig.FrequencyGetHeaderMs + time.Sleep(time.Duration(relayConfig.FrequencyGetHeaderMs) * time.Millisecond) + } else { + break } - - // Verify the relay signature in the relay response - if !config.SkipRelaySignatureCheck { - ok, err := checkRelaySignature(bid, m.builderSigningDomain, relay.PublicKey) - if err != nil { - log.WithError(err).Error("error verifying relay signature") - return - } - if !ok { - log.Error("failed to verify relay signature") - return + } + wg.Wait() + + // select only the bid which was most recently received + if len(bidResults) > 0 { + log.WithField("totalBids", len(bidResults)).Debug("received headers from relay via timing games") + var latestBid *builderSpec.VersionedSignedBuilderBid + var latestContentType string + var latestTime time.Time + for _, br := range bidResults { + if latestBid == nil || br.timestamp.After(latestTime) { + latestBid = br.bid + latestContentType = br.contentType + latestTime = br.timestamp } } + return latestBid, latestContentType + } + log.Warn("no headers received via timing games") + return nil, "" + } - // Verify response coherence with proposer's input data - if bidInfo.parentHash.String() != parentHashHex { - log.WithFields(logrus.Fields{ - "originalParentHash": parentHashHex, - "responseParentHash": bidInfo.parentHash.String(), - }).Error("proposer and relay parent hashes are not the same") - return - } + // in the case if frequency is not set, send only one getHeader request + return m.sendGetHeaderRequest(log, relay, url, slotUID, userAgent, proposerAcceptContentTypes) +} - // Ignore bids with 0 value - isZeroValue := bidInfo.value.IsZero() - isEmptyListTxRoot := bidInfo.txRoot.String() == "0x7ffe241ea60187fdb0187bfa22de35d1f9bed7ab061d9401fd47e34a54fbede1" - if isZeroValue || isEmptyListTxRoot { - log.Warn("ignoring bid with 0 value") - return - } +// sendGetHeaderRequest sends a single getHeader request to a relay and returns the bid and content type +func (m *BoostService) sendGetHeaderRequest( + log *logrus.Entry, + relay types.RelayEntry, + url string, + slotUID uuid.UUID, + userAgent string, + proposerAcceptContentTypes string, +) (*builderSpec.VersionedSignedBuilderBid, string) { + // Make a new request + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + log.WithError(err).Warn("error creating new request") + return nil, "" + } - log.Debug("bid received") + // Add header fields to this request + req.Header.Set(HeaderAccept, proposerAcceptContentTypes) + req.Header.Set(HeaderKeySlotUID, slotUID.String()) + req.Header.Set(HeaderUserAgent, userAgent) + req.Header.Set(HeaderDateMilliseconds, fmt.Sprintf("%d", time.Now().UTC().UnixMilli())) + req.Header.Set(HeaderTimeoutMs, strconv.FormatInt(m.httpClientGetHeader.Timeout.Milliseconds(), 10)) + + // Send the request + log.Debug("requesting header") + start := time.Now() + resp, err := m.httpClientGetHeader.Do(req) + RecordRelayLatency(params.PathGetHeader, relay.GetID(), float64(time.Since(start).Microseconds())) + if err != nil { + log.WithError(err).Warn("error calling getHeader on relay") + return nil, "" + } + defer resp.Body.Close() - RecordRelayLastSlot(relay.String(), uint64(slot)) + RecordRelayStatusCode(strconv.Itoa(resp.StatusCode), params.PathGetHeader, relay.GetID()) - valueEthFloat64, _ := valueEth.Float64() - RecordBidValue(relay.String(), valueEthFloat64) + // Check if no header is available + if resp.StatusCode == http.StatusNoContent { + log.Debug("no-content response") + return nil, "" + } - // Skip if value is lower than the minimum bid - if bidInfo.value.CmpBig(m.relayMinBid.BigInt()) == -1 { - log.Debug("ignoring bid below min-bid value") - IncrementBidBelowMinBid(relay.String()) - return - } + // Check that the response was successful + if resp.StatusCode != http.StatusOK { + err = fmt.Errorf("%w: %d", errHTTPErrorResponse, resp.StatusCode) + log.WithError(err).Warn("error status code") + return nil, "" + } - mu.Lock() - defer mu.Unlock() - - // Create a copy of the relay instance with its encoding preference. If we request SSZ and the relay - // responds with JSON, we know that it does not support SSZ yet. This preference will be used in getPayload, - // because we must encode the blinded block in the request in such a way that the relay can decode it. - relayWithEncodingPreference := relay.Copy() - relayWithEncodingPreference.SupportsSSZ = respContentType == MediaTypeOctetStream - - // Remember which relays delivered which bids (multiple relays might deliver the top bid) - relays[BlockHashHex(bidInfo.blockHash.String())] = append(relays[BlockHashHex(bidInfo.blockHash.String())], relayWithEncodingPreference) - - // Compare the bid with already known top bid (if any) - if !result.response.IsEmpty() { - valueDiff := bidInfo.value.Cmp(result.bidInfo.value) - switch valueDiff { - case -1: - // The current bid is less profitable than already known one - log.Debug("ignoring less profitable bid") - return - case 0: - // The current bid is equally profitable as already known one - // Use hash as tiebreaker - previousBidBlockHash := result.bidInfo.blockHash - if bidInfo.blockHash.String() >= previousBidBlockHash.String() { - log.Debug("equally profitable bid lost tiebreaker") - return - } - } - } + // Get the resp body content + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + log.WithError(err).Warn("error reading response body") + return nil, "" + } - // Use this relay's response as mev-boost response because it's most profitable - log.Debug("new best bid") - result.response = *bid - result.bidInfo = bidInfo + // Get the response's content type, default to JSON + respContentType, _, err := mime.ParseMediaType(resp.Header.Get(HeaderContentType)) + if err != nil { + log.WithError(err).Warn("error parsing response content type") + respContentType = MediaTypeJSON + } + log = log.WithField("respContentType", respContentType) - result.t = time.Now() - }(relay) + // Get the optional version, used with SSZ decoding + respEthConsensusVersion := resp.Header.Get(HeaderEthConsensusVersion) + log = log.WithField("respEthConsensusVersion", respEthConsensusVersion) + + // Decode bid + bid := new(builderSpec.VersionedSignedBuilderBid) + err = decodeBid(respBytes, respContentType, respEthConsensusVersion, bid) + if err != nil { + log.WithError(err).Warn("error decoding bid") + return nil, "" } - wg.Wait() - // Set the winning relays before returning - result.relays = relays[BlockHashHex(result.bidInfo.blockHash.String())] - if len(result.relays) > 0 { - RecordWinningBidValue(result.bidInfo.value.Float64()) + // Skip if bid is empty + if bid.IsEmpty() { + log.Debug("skipping empty bid") + return nil, "" } - return result, nil + return bid, respContentType +} + +// processBid validates and stores a bid if it's better than the current best +func (m *BoostService) processBid( + log *logrus.Entry, + relay types.RelayEntry, + bid *builderSpec.VersionedSignedBuilderBid, + respContentType string, + parentHashHex string, + result *bidResp, + relays map[BlockHashHex][]types.RelayEntry, + slot phase0.Slot, +) { + // Getting the bid info will check if there are missing fields in the response + bidInfo, err := parseBidInfo(bid) + if err != nil { + log.WithError(err).Warn("error parsing bid info") + return + } + + // Ignore bids with an empty block + if bidInfo.blockHash == nilHash { + log.Warn("relay responded with empty block hash") + return + } + + // Add some info about the bid to the logger + valueEth := weiBigIntToEthBigFloat(bidInfo.value.ToBig()) + log = log.WithFields(logrus.Fields{ + "blockNumber": bidInfo.blockNumber, + "blockHash": bidInfo.blockHash.String(), + "txRoot": bidInfo.txRoot.String(), + "value": valueEth.Text('f', 18), + }) + + // Ensure the bid uses the correct public key + if relay.PublicKey.String() != bidInfo.pubkey.String() { + log.Errorf("bid pubkey mismatch. expected: %s - got: %s", relay.PublicKey.String(), bidInfo.pubkey.String()) + return + } + + // Verify the relay signature in the relay response + if !config.SkipRelaySignatureCheck { + ok, err := checkRelaySignature(bid, m.builderSigningDomain, relay.PublicKey) + if err != nil { + log.WithError(err).Error("error verifying relay signature") + return + } + if !ok { + log.Error("failed to verify relay signature") + return + } + } + + // Verify response coherence with proposer's input data + if bidInfo.parentHash.String() != parentHashHex { + log.WithFields(logrus.Fields{ + "originalParentHash": parentHashHex, + "responseParentHash": bidInfo.parentHash.String(), + }).Error("proposer and relay parent hashes are not the same") + return + } + + // Ignore bids with 0 value + isZeroValue := bidInfo.value.IsZero() + isEmptyListTxRoot := bidInfo.txRoot.String() == "0x7ffe241ea60187fdb0187bfa22de35d1f9bed7ab061d9401fd47e34a54fbede1" + if isZeroValue || isEmptyListTxRoot { + log.Warn("ignoring bid with 0 value") + return + } + + log.Debug("bid received") + + RecordRelayLastSlot(relay.GetID(), uint64(slot)) + + valueEthFloat64, _ := valueEth.Float64() + RecordBidValue(relay.GetID(), valueEthFloat64) + + // Skip if value is lower than the minimum bid + if bidInfo.value.CmpBig(m.relayMinBid.BigInt()) == -1 { + log.Debug("ignoring bid below min-bid value") + IncrementBidBelowMinBid(relay.GetID()) + return + } + + // Create a copy of the relay instance with its encoding preference. If we request SSZ and the relay + // responds with JSON, we know that it does not support SSZ yet. This preference will be used in getPayload, + // because we must encode the blinded block in the request in such a way that the relay can decode it. + relayWithEncodingPreference := relay.Copy() + relayWithEncodingPreference.SupportsSSZ = respContentType == MediaTypeOctetStream + + // Remember which relays delivered which bids (multiple relays might deliver the top bid) + relays[BlockHashHex(bidInfo.blockHash.String())] = append(relays[BlockHashHex(bidInfo.blockHash.String())], relayWithEncodingPreference) + + // Compare the bid with already known top bid (if any) + if !result.response.IsEmpty() { + valueDiff := bidInfo.value.Cmp(result.bidInfo.value) + switch valueDiff { + case -1: + // The current bid is less profitable than already known one + log.Debug("ignoring less profitable bid") + return + case 0: + // The current bid is equally profitable as already known one + // Use hash as tiebreaker + previousBidBlockHash := result.bidInfo.blockHash + if bidInfo.blockHash.String() >= previousBidBlockHash.String() { + log.Debug("equally profitable bid lost tiebreaker") + return + } + } + } + + // Use this relay's response as mev-boost response because it's most profitable + log.Debug("new best bid") + result.response = *bid + result.bidInfo = bidInfo + + result.t = time.Now() } // decodeBid decodes a bid by SSZ or JSON, depending on the provided respContentType diff --git a/server/get_payload.go b/server/get_payload.go index b298a65b..3ed7fc57 100644 --- a/server/get_payload.go +++ b/server/get_payload.go @@ -143,7 +143,7 @@ func (m *BoostService) innerGetPayload(log *logrus.Entry, signedBlindedBeaconBlo } // Prepare for requests - resultCh := make(chan payloadResult, len(m.relays)) + resultCh := make(chan payloadResult, len(m.relayConfigs)) var received atomic.Bool go func() { // Make sure we receive a response within the timeout @@ -155,7 +155,7 @@ func (m *BoostService) innerGetPayload(log *logrus.Entry, signedBlindedBeaconBlo requestCtx, requestCtxCancel := context.WithTimeout(context.Background(), m.httpClientGetPayload.Timeout) defer requestCtxCancel() - for _, relay := range m.relays { + for _, relayConfig := range m.relayConfigs { go func(relay types.RelayEntry) { var url string if version == GetPayloadV1 { @@ -223,13 +223,13 @@ func (m *BoostService) innerGetPayload(log *logrus.Entry, signedBlindedBeaconBlo log.Debug("submitting signed blinded block") start := time.Now() resp, err := m.httpClientGetPayload.Do(req) - RecordRelayLatency(endpoint, relay.String(), float64(time.Since(start).Microseconds())) + RecordRelayLatency(endpoint, relay.GetID(), float64(time.Since(start).Microseconds())) if err != nil { log.WithError(err).Warnf("error calling getPayload%s on relay", version) return nil, err } - RecordRelayStatusCode(strconv.Itoa(statusCode), endpoint, relay.String()) + RecordRelayStatusCode(strconv.Itoa(statusCode), endpoint, relay.GetID()) // Check that the response was successful if resp.StatusCode != statusCode { err = fmt.Errorf("%w: %d", errHTTPErrorResponse, resp.StatusCode) @@ -296,7 +296,7 @@ func (m *BoostService) innerGetPayload(log *logrus.Entry, signedBlindedBeaconBlo } else { log.Trace("discarding response, already received a correct response") } - }(relay) + }(relayConfig.RelayEntry) } // Wait for the first request to complete diff --git a/server/mock/mock_relay.go b/server/mock/mock_relay.go index a806bb52..3ce8b9d0 100644 --- a/server/mock/mock_relay.go +++ b/server/mock/mock_relay.go @@ -488,6 +488,13 @@ func (m *Relay) OverrideHandleRegisterValidator(method func(w http.ResponseWrite m.handlerOverrideRegisterValidator = method } +func (m *Relay) OverrideHandleGetHeader(method func(w http.ResponseWriter, req *http.Request)) { + m.mu.Lock() + defer m.mu.Unlock() + + m.handlerOverrideGetHeader = method +} + func (m *Relay) OverrideHandleGetPayload(method func(w http.ResponseWriter, req *http.Request)) { m.mu.Lock() defer m.mu.Unlock() diff --git a/server/register_validator.go b/server/register_validator.go index cb31fd10..6ca82352 100644 --- a/server/register_validator.go +++ b/server/register_validator.go @@ -14,16 +14,16 @@ import ( ) func (m *BoostService) registerValidator(log *logrus.Entry, regBytes []byte, header http.Header) error { - respErrCh := make(chan error, len(m.relays)) + respErrCh := make(chan error, len(m.relayConfigs)) log.WithFields(logrus.Fields{ "timeout": m.httpClientRegVal.Timeout, - "numRelays": len(m.relays), + "numRelays": len(m.relayConfigs), "regBytes": len(regBytes), }).Info("calling registerValidator on relays") // Forward request to each relay - for _, relay := range m.relays { + for _, relayConfig := range m.relayConfigs { go func(relay types.RelayEntry) { // Get the URL for this relay requestURL := relay.GetURI(params.PathRegisterValidator) @@ -49,7 +49,7 @@ func (m *BoostService) registerValidator(log *logrus.Entry, regBytes []byte, hea // Send the request start := time.Now() resp, err := m.httpClientRegVal.Do(req) - RecordRelayLatency(params.PathRegisterValidator, relay.String(), float64(time.Since(start).Microseconds())) + RecordRelayLatency(params.PathRegisterValidator, relay.GetID(), float64(time.Since(start).Microseconds())) if err != nil { log.WithError(err).Warn("error calling registerValidator on relay") respErrCh <- err @@ -57,7 +57,7 @@ func (m *BoostService) registerValidator(log *logrus.Entry, regBytes []byte, hea } resp.Body.Close() - RecordRelayStatusCode(strconv.Itoa(resp.StatusCode), params.PathRegisterValidator, relay.String()) + RecordRelayStatusCode(strconv.Itoa(resp.StatusCode), params.PathRegisterValidator, relay.GetID()) // Check if response is successful if resp.StatusCode == http.StatusOK { log.Debug("relay accepted registrations") @@ -68,11 +68,11 @@ func (m *BoostService) registerValidator(log *logrus.Entry, regBytes []byte, hea }).Debug("received an error response from relay") respErrCh <- fmt.Errorf("%w: %d", errHTTPErrorResponse, resp.StatusCode) } - }(relay) + }(relayConfig.RelayEntry) } // Return OK if any relay responds OK - for range m.relays { + for range m.relayConfigs { respErr := <-respErrCh if respErr == nil { // Goroutines are independent, so if there are a lot of configured diff --git a/server/register_validator_test.go b/server/register_validator_test.go index 381fea04..1ffaf5aa 100644 --- a/server/register_validator_test.go +++ b/server/register_validator_test.go @@ -23,7 +23,7 @@ func TestHandleRegisterValidator_EmptyList(t *testing.T) { defer relay.Server.Close() m := &BoostService{ - relays: []types.RelayEntry{relay.RelayEntry}, + relayConfigs: []types.RelayConfig{types.NewRelayConfig(relay.RelayEntry)}, httpClientRegVal: *http.DefaultClient, log: logrus.NewEntry(logrus.New()), } @@ -47,7 +47,7 @@ func TestHandleRegisterValidator_NotEmptyList(t *testing.T) { defer relay.Server.Close() m := &BoostService{ - relays: []types.RelayEntry{relay.RelayEntry}, + relayConfigs: []types.RelayConfig{types.NewRelayConfig(relay.RelayEntry)}, httpClientRegVal: *http.DefaultClient, log: logrus.NewEntry(logrus.New()), } @@ -87,7 +87,7 @@ func TestHandleRegisterValidator_InvalidJSON(t *testing.T) { defer relay.Server.Close() m := &BoostService{ - relays: []types.RelayEntry{relay.RelayEntry}, + relayConfigs: []types.RelayConfig{types.NewRelayConfig(relay.RelayEntry)}, httpClientRegVal: *http.DefaultClient, log: logrus.NewEntry(logrus.New()), } @@ -111,7 +111,7 @@ func TestHandleRegisterValidator_ValidSSZ(t *testing.T) { defer relay.Server.Close() m := &BoostService{ - relays: []types.RelayEntry{relay.RelayEntry}, + relayConfigs: []types.RelayConfig{types.NewRelayConfig(relay.RelayEntry)}, httpClientRegVal: *http.DefaultClient, log: logrus.NewEntry(logrus.New()), } @@ -153,7 +153,7 @@ func TestHandleRegisterValidator_InvalidSSZ(t *testing.T) { defer relay.Server.Close() m := &BoostService{ - relays: []types.RelayEntry{relay.RelayEntry}, + relayConfigs: []types.RelayConfig{types.NewRelayConfig(relay.RelayEntry)}, httpClientRegVal: *http.DefaultClient, log: logrus.NewEntry(logrus.New()), } @@ -183,7 +183,7 @@ func TestHandleRegisterValidator_MultipleRelaysOneSuccess(t *testing.T) { defer relaySuccess.Server.Close() m := &BoostService{ - relays: []types.RelayEntry{badRelay.RelayEntry, relaySuccess.RelayEntry}, + relayConfigs: []types.RelayConfig{types.NewRelayConfig(badRelay.RelayEntry), types.NewRelayConfig(relaySuccess.RelayEntry)}, httpClientRegVal: *http.DefaultClient, log: logrus.NewEntry(logrus.New()), } @@ -218,7 +218,7 @@ func TestHandleRegisterValidator_AllFail(t *testing.T) { }) m := &BoostService{ - relays: []types.RelayEntry{badRelay1.RelayEntry, badRelay2.RelayEntry}, + relayConfigs: []types.RelayConfig{types.NewRelayConfig(badRelay1.RelayEntry), types.NewRelayConfig(badRelay2.RelayEntry)}, httpClientRegVal: *http.DefaultClient, log: logrus.NewEntry(logrus.New()), } @@ -244,7 +244,7 @@ func TestHandleRegisterValidator_RelayNetworkError(t *testing.T) { relay.Server.Close() // simulate network error m := &BoostService{ - relays: []types.RelayEntry{relay.RelayEntry}, + relayConfigs: []types.RelayConfig{types.NewRelayConfig(relay.RelayEntry)}, httpClientRegVal: *http.DefaultClient, log: logrus.NewEntry(logrus.New()), } @@ -272,7 +272,7 @@ func TestHandleRegisterValidator_HeaderPropagation(t *testing.T) { }) m := &BoostService{ - relays: []types.RelayEntry{relay.RelayEntry}, + relayConfigs: []types.RelayConfig{types.NewRelayConfig(relay.RelayEntry)}, httpClientRegVal: *http.DefaultClient, log: logrus.NewEntry(logrus.New()), } diff --git a/server/service.go b/server/service.go index 172535bd..701211bb 100644 --- a/server/service.go +++ b/server/service.go @@ -54,7 +54,7 @@ type slotUID struct { type BoostServiceOpts struct { Log *logrus.Entry ListenAddr string - Relays []types.RelayEntry + RelayConfigs []types.RelayConfig GenesisForkVersionHex string GenesisTime uint64 RelayCheck bool @@ -65,18 +65,21 @@ type BoostServiceOpts struct { RequestTimeoutRegVal time.Duration RequestMaxRetries int + TimeoutGetHeaderMs uint64 + LateInSlotTimeMs uint64 + MetricsAddr string } // BoostService - the mev-boost service type BoostService struct { - listenAddr string - relays []types.RelayEntry - log *logrus.Entry - srv *http.Server - relayCheck bool - relayMinBid types.U256Str - genesisTime uint64 + listenAddr string + relayConfigs []types.RelayConfig + log *logrus.Entry + srv *http.Server + relayCheck bool + relayMinBid types.U256Str + genesisTime uint64 builderSigningDomain phase0.Domain httpClientGetHeader http.Client @@ -84,6 +87,9 @@ type BoostService struct { httpClientRegVal http.Client requestMaxRetries int + timeoutGetHeaderMs uint64 + lateInSlotTimeMs uint64 + bids map[string]bidResp // keeping track of bids, to log the originating relay on withholding bidsLock sync.Mutex @@ -95,7 +101,7 @@ type BoostService struct { // NewBoostService created a new BoostService func NewBoostService(opts BoostServiceOpts) (*BoostService, error) { - if len(opts.Relays) == 0 { + if len(opts.RelayConfigs) == 0 { return nil, errNoRelays } @@ -105,15 +111,15 @@ func NewBoostService(opts BoostServiceOpts) (*BoostService, error) { } return &BoostService{ - listenAddr: opts.ListenAddr, - relays: opts.Relays, - log: opts.Log, - relayCheck: opts.RelayCheck, - relayMinBid: opts.RelayMinBid, - genesisTime: opts.GenesisTime, - bids: make(map[string]bidResp), - slotUID: &slotUID{}, - metricsAddr: opts.MetricsAddr, + listenAddr: opts.ListenAddr, + relayConfigs: opts.RelayConfigs, + log: opts.Log, + relayCheck: opts.RelayCheck, + relayMinBid: opts.RelayMinBid, + genesisTime: opts.GenesisTime, + bids: make(map[string]bidResp), + slotUID: &slotUID{}, + metricsAddr: opts.MetricsAddr, builderSigningDomain: builderSigningDomain, httpClientGetHeader: http.Client{ @@ -128,7 +134,9 @@ func NewBoostService(opts BoostServiceOpts) (*BoostService, error) { Timeout: opts.RequestTimeoutRegVal, CheckRedirect: httpClientDisallowRedirects, }, - requestMaxRetries: opts.RequestMaxRetries, + requestMaxRetries: opts.RequestMaxRetries, + timeoutGetHeaderMs: opts.TimeoutGetHeaderMs, + lateInSlotTimeMs: opts.LateInSlotTimeMs, }, nil } @@ -280,6 +288,7 @@ func (m *BoostService) handleGetHeader(w http.ResponseWriter, req *http.Request) rawProposerAcceptContentTypes = req.Header.Get(HeaderAccept) parsedProposerAcceptContentTypes = goacceptheaders.Parse(rawProposerAcceptContentTypes) + headerTimeoutString = req.Header.Get(HeaderTimeoutMs) ) // Parse the slot @@ -302,8 +311,17 @@ func (m *BoostService) handleGetHeader(w http.ResponseWriter, req *http.Request) }) log.Debug("handling request") + if headerTimeoutString == "" { + headerTimeoutString = "0" + } + + headerTimeout, err := strconv.ParseUint(headerTimeoutString, 10, 64) + if err != nil { + m.respondError(w, http.StatusBadRequest, err.Error()) + return + } // Query the relays for the header - result, err := m.getHeader(log, slot, pubkey, parentHashHex, ua, rawProposerAcceptContentTypes) + result, err := m.getHeader(log, slot, pubkey, parentHashHex, ua, rawProposerAcceptContentTypes, headerTimeout) if err != nil { IncrementBeaconNodeStatus(strconv.Itoa(http.StatusBadRequest), params.PathGetHeader) m.respondError(w, http.StatusBadRequest, err.Error()) @@ -487,7 +505,7 @@ func (m *BoostService) CheckRelays() int { var wg sync.WaitGroup var numSuccessRequestsToRelay uint32 - for _, r := range m.relays { + for _, relayConfig := range m.relayConfigs { wg.Add(1) go func(relay types.RelayEntry) { @@ -498,12 +516,12 @@ func (m *BoostService) CheckRelays() int { start := time.Now() code, err := SendHTTPRequest(context.Background(), m.httpClientGetHeader, http.MethodGet, url, "", nil, nil, nil) - RecordRelayLatency(params.PathStatus, relay.String(), float64(time.Since(start).Microseconds())) + RecordRelayLatency(params.PathStatus, relay.GetID(), float64(time.Since(start).Microseconds())) if err != nil { log.WithError(err).Error("relay status error - request failed") return } - RecordRelayStatusCode(strconv.Itoa(code), params.PathStatus, relay.String()) + RecordRelayStatusCode(strconv.Itoa(code), params.PathStatus, relay.GetID()) if code == http.StatusOK { log.Debug("relay status OK") } else { @@ -513,7 +531,7 @@ func (m *BoostService) CheckRelays() int { // Success: increase counter and cancel all pending requests to other relays atomic.AddUint32(&numSuccessRequestsToRelay, 1) - }(r) + }(relayConfig.RelayEntry) } // At the end, wait for every routine and return status according to relay's ones. diff --git a/server/service_test.go b/server/service_test.go index 06c4002b..a1480c32 100644 --- a/server/service_test.go +++ b/server/service_test.go @@ -55,17 +55,17 @@ func newTestBackend(t *testing.T, numRelays int, relayTimeout time.Duration) *te relays: make([]*mock.Relay, numRelays), } - relayEntries := make([]types.RelayEntry, numRelays) + relayConfigs := make([]types.RelayConfig, numRelays) for i := 0; i < numRelays; i++ { // Create a mock relay backend.relays[i] = mock.NewRelay(t) - relayEntries[i] = backend.relays[i].RelayEntry + relayConfigs[i] = types.NewRelayConfig(backend.relays[i].RelayEntry) } opts := BoostServiceOpts{ Log: mock.TestLog, ListenAddr: "localhost:12345", - Relays: relayEntries, + RelayConfigs: relayConfigs, GenesisForkVersionHex: "0x00000000", RelayCheck: true, RelayMinBid: types.IntToU256(12345), @@ -73,6 +73,8 @@ func newTestBackend(t *testing.T, numRelays int, relayTimeout time.Duration) *te RequestTimeoutGetPayload: relayTimeout, RequestTimeoutRegVal: relayTimeout, RequestMaxRetries: 5, + TimeoutGetHeaderMs: 900, + LateInSlotTimeMs: 1000, } service, err := NewBoostService(opts) require.NoError(t, err) @@ -124,7 +126,7 @@ func TestNewBoostServiceErrors(t *testing.T) { _, err := NewBoostService(BoostServiceOpts{ Log: mock.TestLog, ListenAddr: ":123", - Relays: []types.RelayEntry{}, + RelayConfigs: []types.RelayConfig{}, GenesisForkVersionHex: "0x00000000", GenesisTime: 0, RelayCheck: true, @@ -502,7 +504,7 @@ func TestGetHeader(t *testing.T) { // Simulate a different public key registered to mev-boost pk := phase0.BLSPubKey{} - backend.boost.relays[0].PublicKey = pk + backend.boost.relayConfigs[0].RelayEntry.PublicKey = pk rr := backend.request(t, http.MethodGet, path, header, nil) require.Equal(t, 1, backend.relays[0].GetRequestCount(path)) @@ -761,6 +763,297 @@ func TestGetHeaderBids(t *testing.T) { }) } +func TestGetHeaderTimingGames(t *testing.T) { + hash := mock.HexToHash("0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7") + pubkey := mock.HexToPubkey( + "0x8a1d7b8dd64e0aafe7ea7b6c95065c9364cf99d38470c12ee807d55f7de1529ad29ce2c422e0b65e3d5a05c02caca249") + path := getHeaderPath(3, hash, pubkey) + + t.Run("Relay with timing games sends multiple requests", func(t *testing.T) { + header := make(http.Header) + header.Set(HeaderAccept, MediaTypeJSON) + + backend := newTestBackend(t, 1, time.Second) + + backend.boost.relayConfigs[0].EnableTimingGames = true + backend.boost.relayConfigs[0].TargetFirstRequestMs = 0 + backend.boost.relayConfigs[0].FrequencyGetHeaderMs = 50 // request every 50ms + + rr := backend.request(t, http.MethodGet, path, header, nil) + require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) + + // should have received multiple requests due to timing games + requestCount := backend.relays[0].GetRequestCount(path) + require.Greater(t, requestCount, 1) + }) + + t.Run("Relay with timing games delays first request", func(t *testing.T) { + header := make(http.Header) + header.Set(HeaderAccept, MediaTypeJSON) + + backend := newTestBackend(t, 1, time.Second) + + backend.boost.relayConfigs[0].EnableTimingGames = true + backend.boost.relayConfigs[0].TargetFirstRequestMs = 100 // wait 100ms from slot start + backend.boost.relayConfigs[0].FrequencyGetHeaderMs = 0 // no multiple requests + + rr := backend.request(t, http.MethodGet, path, header, nil) + + require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) + // with no frequency, should only send one request + require.Equal(t, 1, backend.relays[0].GetRequestCount(path)) + }) + + t.Run("Mix of timing games and normal relays", func(t *testing.T) { + header := make(http.Header) + header.Set(HeaderAccept, MediaTypeJSON) + + backend := newTestBackend(t, 3, time.Second) + + // timing games enabled for only first relay + backend.boost.relayConfigs[0].EnableTimingGames = true + backend.boost.relayConfigs[0].TargetFirstRequestMs = 0 + backend.boost.relayConfigs[0].FrequencyGetHeaderMs = 30 + + // second relay: without timing games enabled + backend.relays[1].GetHeaderResponse = backend.relays[1].MakeGetHeaderResponse( + 12346, + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0x8a1d7b8dd64e0aafe7ea7b6c95065c9364cf99d38470c12ee807d55f7de1529ad29ce2c422e0b65e3d5a05c02caca249", + spec.DataVersionDeneb, + ) + + // third relay: without timing games enabled + backend.relays[2].GetHeaderResponse = backend.relays[2].MakeGetHeaderResponse( + 12347, + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0x8a1d7b8dd64e0aafe7ea7b6c95065c9364cf99d38470c12ee807d55f7de1529ad29ce2c422e0b65e3d5a05c02caca249", + spec.DataVersionDeneb, + ) + + rr := backend.request(t, http.MethodGet, path, header, nil) + require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) + + // relay1 should have received more then 1 request due to timing games + require.Greater(t, backend.relays[0].GetRequestCount(path), 1) + // relay 2 and 3 should have received exactly one request + require.Equal(t, 1, backend.relays[1].GetRequestCount(path)) + require.Equal(t, 1, backend.relays[2].GetRequestCount(path)) + + resp := new(builderSpec.VersionedSignedBuilderBid) + err := json.Unmarshal(rr.Body.Bytes(), resp) + require.NoError(t, err) + value, err := resp.Value() + require.NoError(t, err) + require.Equal(t, uint256.NewInt(12347), value) + }) + + t.Run("Timing games relay with higher bid wins", func(t *testing.T) { + header := make(http.Header) + header.Set(HeaderAccept, MediaTypeJSON) + + backend := newTestBackend(t, 2, time.Second) + + // relay1: timing games with higher bid + backend.boost.relayConfigs[0].EnableTimingGames = true + backend.boost.relayConfigs[0].TargetFirstRequestMs = 0 + backend.boost.relayConfigs[0].FrequencyGetHeaderMs = 40 + backend.relays[0].GetHeaderResponse = backend.relays[0].MakeGetHeaderResponse( + 12350, + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0x8a1d7b8dd64e0aafe7ea7b6c95065c9364cf99d38470c12ee807d55f7de1529ad29ce2c422e0b65e3d5a05c02caca249", + spec.DataVersionDeneb, + ) + + // relay2: normal with lower bid + backend.relays[1].GetHeaderResponse = backend.relays[1].MakeGetHeaderResponse( + 12348, + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0x8a1d7b8dd64e0aafe7ea7b6c95065c9364cf99d38470c12ee807d55f7de1529ad29ce2c422e0b65e3d5a05c02caca249", + spec.DataVersionDeneb, + ) + + rr := backend.request(t, http.MethodGet, path, header, nil) + require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) + + // relay1's bid should win + resp := new(builderSpec.VersionedSignedBuilderBid) + err := json.Unmarshal(rr.Body.Bytes(), resp) + require.NoError(t, err) + value, err := resp.Value() + require.NoError(t, err) + require.Equal(t, uint256.NewInt(12350), value) + }) + + t.Run("Timing games with SSZ encoding", func(t *testing.T) { + header := make(http.Header) + header.Set(HeaderEthConsensusVersion, "deneb") + header.Set(HeaderAccept, MediaTypeOctetStream) + + backend := newTestBackend(t, 1, time.Second) + backend.boost.relayConfigs[0].EnableTimingGames = true + backend.boost.relayConfigs[0].TargetFirstRequestMs = 0 + backend.boost.relayConfigs[0].FrequencyGetHeaderMs = 50 + + backend.relays[0].ForceSSZ = true + backend.relays[0].GetHeaderResponse = backend.relays[0].MakeGetHeaderResponse( + 12345, + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0x8a1d7b8dd64e0aafe7ea7b6c95065c9364cf99d38470c12ee807d55f7de1529ad29ce2c422e0b65e3d5a05c02caca249", + spec.DataVersionDeneb, + ) + + rr := backend.request(t, http.MethodGet, path, header, nil) + require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) + require.Greater(t, backend.relays[0].GetRequestCount(path), 1) + require.Equal(t, MediaTypeOctetStream, rr.Header().Get(HeaderContentType)) + + bid := new(builderApiDeneb.SignedBuilderBid) + err := bid.UnmarshalSSZ(rr.Body.Bytes()) + require.NoError(t, err) + }) + + t.Run("Timing games respects timeout budget", func(t *testing.T) { + header := make(http.Header) + header.Set(HeaderAccept, MediaTypeJSON) + + backend := newTestBackend(t, 1, time.Second) + + backend.boost.timeoutGetHeaderMs = 100 + backend.boost.lateInSlotTimeMs = 1000 + + backend.boost.relayConfigs[0].EnableTimingGames = true + backend.boost.relayConfigs[0].TargetFirstRequestMs = 0 + backend.boost.relayConfigs[0].FrequencyGetHeaderMs = 20 + + start := time.Now() + rr := backend.request(t, http.MethodGet, path, header, nil) + elapsed := time.Since(start) + + require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) + require.LessOrEqual(t, elapsed.Milliseconds(), int64(300)) + + requestCount := backend.relays[0].GetRequestCount(path) + require.Greater(t, requestCount, 1) + require.Equal(t, 5, requestCount) // 100ms / 20ms = 5 requests + }) + + t.Run("Multiple timing games relays compete", func(t *testing.T) { + header := make(http.Header) + header.Set(HeaderAccept, MediaTypeJSON) + + backend := newTestBackend(t, 2, time.Second) + + // both relays use timing games + backend.boost.relayConfigs[0].EnableTimingGames = true + backend.boost.relayConfigs[0].TargetFirstRequestMs = 0 + backend.boost.relayConfigs[0].FrequencyGetHeaderMs = 40 + backend.relays[0].GetHeaderResponse = backend.relays[0].MakeGetHeaderResponse( + 12345, + "0xa18385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0x8a1d7b8dd64e0aafe7ea7b6c95065c9364cf99d38470c12ee807d55f7de1529ad29ce2c422e0b65e3d5a05c02caca249", + spec.DataVersionDeneb, + ) + + backend.boost.relayConfigs[1].EnableTimingGames = true + backend.boost.relayConfigs[1].TargetFirstRequestMs = 0 + backend.boost.relayConfigs[1].FrequencyGetHeaderMs = 35 + backend.relays[1].GetHeaderResponse = backend.relays[1].MakeGetHeaderResponse( + 12345, + "0xa28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0x8a1d7b8dd64e0aafe7ea7b6c95065c9364cf99d38470c12ee807d55f7de1529ad29ce2c422e0b65e3d5a05c02caca249", + spec.DataVersionDeneb, + ) + + rr := backend.request(t, http.MethodGet, path, header, nil) + require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) + + require.Greater(t, backend.relays[0].GetRequestCount(path), 1) + require.Greater(t, backend.relays[1].GetRequestCount(path), 1) + + resp := new(builderSpec.VersionedSignedBuilderBid) + err := json.Unmarshal(rr.Body.Bytes(), resp) + require.NoError(t, err) + blockHash, err := resp.BlockHash() + require.NoError(t, err) + require.Equal(t, "0xa18385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", blockHash.String()) + }) + + t.Run("Higher bid received on later request wins", func(t *testing.T) { + header := make(http.Header) + header.Set(HeaderAccept, MediaTypeJSON) + + backend := newTestBackend(t, 1, time.Second) + + // enable timing games for relay1 + backend.boost.relayConfigs[0].EnableTimingGames = true + backend.boost.relayConfigs[0].TargetFirstRequestMs = 0 + backend.boost.relayConfigs[0].FrequencyGetHeaderMs = 50 // request every 50ms + + requestCount := 0 + + backend.relays[0].OverrideHandleGetHeader(func(w http.ResponseWriter, _ *http.Request) { + requestCount++ + + var resp *builderSpec.VersionedSignedBuilderBid + switch requestCount { + case 1: + // first request: lower bid + resp = backend.relays[0].MakeGetHeaderResponse( + 12345, + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0x8a1d7b8dd64e0aafe7ea7b6c95065c9364cf99d38470c12ee807d55f7de1529ad29ce2c422e0b65e3d5a05c02caca249", + spec.DataVersionDeneb, + ) + case 2: + // second request: medium bid + resp = backend.relays[0].MakeGetHeaderResponse( + 12400, + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0x8a1d7b8dd64e0aafe7ea7b6c95065c9364cf99d38470c12ee807d55f7de1529ad29ce2c422e0b65e3d5a05c02caca249", + spec.DataVersionDeneb, + ) + default: + resp = backend.relays[0].MakeGetHeaderResponse( + 12500, + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0xe28385e7bd68df656cd0042b74b69c3104b5356ed1f20eb69f1f925df47a3ab7", + "0x8a1d7b8dd64e0aafe7ea7b6c95065c9364cf99d38470c12ee807d55f7de1529ad29ce2c422e0b65e3d5a05c02caca249", + spec.DataVersionDeneb, + ) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(resp); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + }) + + rr := backend.request(t, http.MethodGet, path, header, nil) + require.Equal(t, http.StatusOK, rr.Code, rr.Body.String()) + + finalCount := backend.relays[0].GetRequestCount(path) + require.Greater(t, finalCount, 2) + + // should select the highest bid from the later request + bidResp := new(builderSpec.VersionedSignedBuilderBid) + err := json.Unmarshal(rr.Body.Bytes(), bidResp) + require.NoError(t, err) + value, err := bidResp.Value() + require.NoError(t, err) + require.Equal(t, uint256.NewInt(12500), value) + }) +} + func TestGetPayload(t *testing.T) { path := params.PathGetPayload blockHash := mock.HexToHash("0x534809bd2b6832edff8d8ce4cb0e50068804fd1ef432c8362ad708a74fdc0e46") @@ -1300,7 +1593,7 @@ func TestCheckRelays(t *testing.T) { url, err := url.ParseRequestURI(backend.relays[0].Server.URL) require.NoError(t, err) - backend.boost.relays[0].URL = url + backend.boost.relayConfigs[0].RelayEntry.URL = url numHealthyRelays := backend.boost.CheckRelays() require.Equal(t, 0, numHealthyRelays) }) diff --git a/server/types/relay_entry.go b/server/types/relay_entry.go index ec97f09f..7bb6336e 100644 --- a/server/types/relay_entry.go +++ b/server/types/relay_entry.go @@ -12,13 +12,34 @@ import ( type RelayEntry struct { PublicKey phase0.BLSPubKey URL *url.URL + ID string SupportsSSZ bool } +type RelayConfig struct { + RelayEntry RelayEntry + EnableTimingGames bool + TargetFirstRequestMs uint64 + FrequencyGetHeaderMs uint64 +} + +func NewRelayConfig(entry RelayEntry) RelayConfig { + return RelayConfig{ + RelayEntry: entry, + } +} + func (r *RelayEntry) String() string { return r.URL.String() } +func (r *RelayEntry) GetID() string { + if r.ID != "" { + return r.ID + } + return r.URL.String() +} + // GetURI returns the full request URI with scheme, host, path and args. func GetURI(url *url.URL, path string) string { u2 := *url @@ -77,6 +98,7 @@ func RelayEntriesToStrings(relays []RelayEntry) []string { // Copy returns a deep copy of the relay entry. func (r *RelayEntry) Copy() (ret RelayEntry) { ret.PublicKey = r.PublicKey + ret.ID = r.ID ret.SupportsSSZ = r.SupportsSSZ if r.URL != nil { urlCopy := *r.URL