From b0a4dfa51b5a9875cc2ee1d7a5bbcac41083efda Mon Sep 17 00:00:00 2001 From: James Scott Date: Wed, 20 Aug 2025 19:18:08 +0000 Subject: [PATCH] feat(web_feature_consumer): Add parser for V3 web-features data The web-features data format is being updated to version 3. This new version changes the structure of the `features` object to support different kinds of feature entries: `feature`, `moved`, and `split`, identified by a `kind` property. This commit introduces a new `V3Parser` to handle this new format. It uses a discriminated union pattern by first peeking at the `kind` property of each feature and then unmarshalling the JSON into the corresponding Go struct. The existing parser for the V2 format has been preserved to handle older data formats. Additionally, comprehensive tests for the new V3 parsing logic have been added to ensure correctness. The V3 tests, where real data is fetched, are currently skipped as the v3.0.0 release is not yet available. --- web_platform_dx__web_features_extras.txt | 6 + .../web_feature_consumer/pkg/data/parser.go | 132 ++++++- .../pkg/data/parser_test.go | 359 ++++++++++++++++-- 3 files changed, 452 insertions(+), 45 deletions(-) diff --git a/web_platform_dx__web_features_extras.txt b/web_platform_dx__web_features_extras.txt index f4037926f..9ba1140ac 100644 --- a/web_platform_dx__web_features_extras.txt +++ b/web_platform_dx__web_features_extras.txt @@ -3,6 +3,12 @@ package web_platform_dx__web_features // This is a temporary file until v3 lands // Takes files generated from https://github.com/GoogleChrome/webstatus.dev/pull/1635 +type FeatureDataKind string + +const ( + Feature FeatureDataKind = "feature" +) + type FeatureMovedDataKind string const ( diff --git a/workflows/steps/services/web_feature_consumer/pkg/data/parser.go b/workflows/steps/services/web_feature_consumer/pkg/data/parser.go index d6ce7ca86..bf11c2e9c 100644 --- a/workflows/steps/services/web_feature_consumer/pkg/data/parser.go +++ b/workflows/steps/services/web_feature_consumer/pkg/data/parser.go @@ -27,18 +27,35 @@ import ( // Parser contains the logic to parse the JSON from the web-features Github Release. type Parser struct{} +// V3Parser contains the logic to parse the JSON from the web-features Github Release. +type V3Parser struct{} + var ErrUnexpectedFormat = errors.New("unexpected format") var ErrUnableToProcess = errors.New("unable to process the data") -// rawWebFeaturesJSONData is used to parse the source JSON. +// rawWebFeaturesJSONDataV2 is used to parse the source JSON. +// It holds the features as raw JSON messages to be processed individually. +type rawWebFeaturesJSONDataV2 struct { + Browsers web_platform_dx__web_features.Browsers `json:"browsers"` + Groups map[string]web_platform_dx__web_features.GroupData `json:"groups"` + Snapshots map[string]web_platform_dx__web_features.SnapshotData `json:"snapshots"` + Features map[string]web_platform_dx__web_features.FeatureValue `json:"features"` +} + +// rawWebFeaturesJSONDataV3 is used to parse the source JSON. // It holds the features as raw JSON messages to be processed individually. -type rawWebFeaturesJSONData struct { +type rawWebFeaturesJSONDataV3 struct { Browsers web_platform_dx__web_features.Browsers `json:"browsers"` Groups map[string]web_platform_dx__web_features.GroupData `json:"groups"` Snapshots map[string]web_platform_dx__web_features.SnapshotData `json:"snapshots"` // TODO: When we move to v3, we will change Features to being json.RawMessage - Features map[string]web_platform_dx__web_features.FeatureValue `json:"features"` + Features json.RawMessage `json:"features"` +} + +// featureKindPeek is a small helper struct to find the discriminator value in V3. +type featureKindPeek struct { + Kind string `json:"kind"` } // Parse expects the raw bytes for a map of string to @@ -47,7 +64,7 @@ type rawWebFeaturesJSONData struct { // It will consume the readcloser and close it. func (p Parser) Parse(in io.ReadCloser) (*webdxfeaturetypes.ProcessedWebFeaturesData, error) { defer in.Close() - var source rawWebFeaturesJSONData + var source rawWebFeaturesJSONDataV2 decoder := json.NewDecoder(in) err := decoder.Decode(&source) if err != nil { @@ -59,7 +76,28 @@ func (p Parser) Parse(in io.ReadCloser) (*webdxfeaturetypes.ProcessedWebFeatures return processedData, nil } -func postProcess(data *rawWebFeaturesJSONData) *webdxfeaturetypes.ProcessedWebFeaturesData { +// Parse expects the raw bytes for a map of string to +// https://github.com/web-platform-dx/web-features/blob/main/schemas/defs.schema.json +// The string is the feature ID. +// It will consume the readcloser and close it. +func (p V3Parser) Parse(in io.ReadCloser) (*webdxfeaturetypes.ProcessedWebFeaturesData, error) { + defer in.Close() + var source rawWebFeaturesJSONDataV3 + decoder := json.NewDecoder(in) + err := decoder.Decode(&source) + if err != nil { + return nil, errors.Join(ErrUnexpectedFormat, err) + } + + processedData, err := postProcessV3(&source) + if err != nil { + return nil, errors.Join(ErrUnableToProcess, err) + } + + return processedData, nil +} + +func postProcess(data *rawWebFeaturesJSONDataV2) *webdxfeaturetypes.ProcessedWebFeaturesData { featureKinds := postProcessFeatureValue(data.Features) return &webdxfeaturetypes.ProcessedWebFeaturesData{ @@ -70,6 +108,90 @@ func postProcess(data *rawWebFeaturesJSONData) *webdxfeaturetypes.ProcessedWebFe } } +func postProcessV3(data *rawWebFeaturesJSONDataV3) (*webdxfeaturetypes.ProcessedWebFeaturesData, error) { + featureKinds, err := postProcessFeatureValueV3(data.Features) + if err != nil { + return nil, err + } + + return &webdxfeaturetypes.ProcessedWebFeaturesData{ + Browsers: data.Browsers, + Groups: data.Groups, + Snapshots: data.Snapshots, + Features: featureKinds, + }, nil +} + +func postProcessFeatureValueV3(data json.RawMessage) (*webdxfeaturetypes.FeatureKinds, error) { + featureKinds := webdxfeaturetypes.FeatureKinds{ + Data: nil, + Moved: nil, + Split: nil, + } + + featureRawMessageMap := make(map[string]json.RawMessage) + + err := json.Unmarshal(data, &featureRawMessageMap) + if err != nil { + return nil, err + } + + for id, rawFeature := range featureRawMessageMap { + // Peek inside the raw JSON to find the "kind" + var peek featureKindPeek + if err := json.Unmarshal(rawFeature, &peek); err != nil { + // Skip or log features that don't have a 'kind' field + continue + } + + // Switch on the explicit "kind" to unmarshal into the correct type + switch peek.Kind { + case string(web_platform_dx__web_features.Feature): + if featureKinds.Data == nil { + featureKinds.Data = make(map[string]web_platform_dx__web_features.FeatureValue) + } + var value web_platform_dx__web_features.FeatureValue + if err := json.Unmarshal(rawFeature, &value); err != nil { + return nil, err + } + // Run your existing post-processing logic + featureKinds.Data[id] = web_platform_dx__web_features.FeatureValue{ + Caniuse: postProcessStringOrStringArray(value.Caniuse), + CompatFeatures: value.CompatFeatures, + Description: value.Description, + DescriptionHTML: value.DescriptionHTML, + Group: postProcessStringOrStringArray(value.Group), + Name: value.Name, + Snapshot: postProcessStringOrStringArray(value.Snapshot), + Spec: postProcessStringOrStringArray(value.Spec), + Status: postProcessStatus(value.Status), + Discouraged: value.Discouraged, + } + + case string(web_platform_dx__web_features.Moved): + if featureKinds.Moved == nil { + featureKinds.Moved = make(map[string]web_platform_dx__web_features.FeatureMovedData) + } + var value web_platform_dx__web_features.FeatureMovedData + if err := json.Unmarshal(rawFeature, &value); err != nil { + return nil, err + } + featureKinds.Moved[id] = value + + case string(web_platform_dx__web_features.Split): + if featureKinds.Split == nil { + featureKinds.Split = make(map[string]web_platform_dx__web_features.FeatureSplitData) + } + var value web_platform_dx__web_features.FeatureSplitData + if err := json.Unmarshal(rawFeature, &value); err != nil { + return nil, err + } + featureKinds.Split[id] = value + } + } + + return &featureKinds, nil +} func postProcessFeatureValue( data map[string]web_platform_dx__web_features.FeatureValue) *webdxfeaturetypes.FeatureKinds { featureKinds := webdxfeaturetypes.FeatureKinds{ diff --git a/workflows/steps/services/web_feature_consumer/pkg/data/parser_test.go b/workflows/steps/services/web_feature_consumer/pkg/data/parser_test.go index 9c08774d0..b66ecca5f 100644 --- a/workflows/steps/services/web_feature_consumer/pkg/data/parser_test.go +++ b/workflows/steps/services/web_feature_consumer/pkg/data/parser_test.go @@ -15,6 +15,7 @@ package data import ( + "encoding/json" "errors" "io" "os" @@ -50,13 +51,52 @@ func TestParse(t *testing.T) { if len(result.Features.Data) == 0 { t.Error("unexpected empty map for features") } - // TODO: When we move to v3, there will be moved and split features - // if len(result.Features.Moved) == 0 { - // t.Error("unexpected empty map for moved features") - // } - // if len(result.Features.Split) == 0 { - // t.Error("unexpected empty map for split features") - // } + if len(result.Features.Moved) != 0 { + t.Error("unexpected map for moved features") + } + if len(result.Features.Split) != 0 { + t.Error("unexpected map for split features") + } + if len(result.Groups) == 0 { + t.Error("unexpected empty map for groups") + } + if len(result.Snapshots) == 0 { + t.Error("unexpected empty map for snapshots") + } + }) + } +} + +func TestParseV3(t *testing.T) { + testCases := []struct { + name string + path string + }{ + { + name: "data.json from https://github.com/web-platform-dx/web-features/releases/tag/v3.0.0", + path: path.Join("testdata", "data.json"), + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Skip("3.0.0 does not exist yet") + file, err := os.Open(tc.path) + if err != nil { + t.Fatalf("unable to read file err %s", err.Error()) + } + result, err := Parser{}.Parse(file) + if err != nil { + t.Errorf("unable to parse file err %s", err.Error()) + } + if len(result.Features.Data) == 0 { + t.Error("unexpected empty map for features") + } + if len(result.Features.Moved) == 0 { + t.Error("unexpected empty map for moved features") + } + if len(result.Features.Split) == 0 { + t.Error("unexpected empty map for split features") + } if len(result.Groups) == 0 { t.Error("unexpected empty map for groups") } @@ -68,21 +108,33 @@ func TestParse(t *testing.T) { } +type parser interface { + Parse(io.ReadCloser) (*webdxfeaturetypes.ProcessedWebFeaturesData, error) +} + func TestParseError(t *testing.T) { testCases := []struct { name string input io.ReadCloser + testParser parser expectedError error }{ { name: "bad format", input: io.NopCloser(strings.NewReader("Hello, world!")), expectedError: ErrUnexpectedFormat, + testParser: Parser{}, + }, + { + name: "bad format", + input: io.NopCloser(strings.NewReader("Hello, world!")), + expectedError: ErrUnexpectedFormat, + testParser: V3Parser{}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - result, err := Parser{}.Parse(tc.input) + result, err := tc.testParser.Parse(tc.input) if !errors.Is(err, tc.expectedError) { t.Errorf("unexpected error expected %v received %v", tc.expectedError, err) } @@ -95,45 +147,49 @@ func TestParseError(t *testing.T) { func valuePtr[T any](in T) *T { return &in } +func testBrowsers() web_platform_dx__web_features.Browsers { + return web_platform_dx__web_features.Browsers{ + Chrome: web_platform_dx__web_features.BrowserData{ + Name: "chrome", + Releases: nil, + }, + ChromeAndroid: web_platform_dx__web_features.BrowserData{ + Name: "chrome_android", + Releases: nil, + }, + Edge: web_platform_dx__web_features.BrowserData{ + Name: "edge", + Releases: nil, + }, + Firefox: web_platform_dx__web_features.BrowserData{ + Name: "firefox", + Releases: nil, + }, + FirefoxAndroid: web_platform_dx__web_features.BrowserData{ + Name: "firefox_android", + Releases: nil, + }, + Safari: web_platform_dx__web_features.BrowserData{ + Name: "safari", + Releases: nil, + }, + SafariIos: web_platform_dx__web_features.BrowserData{ + Name: "safari_ios", + Releases: nil, + }, + } +} + func TestPostProcess(t *testing.T) { testCases := []struct { name string - featureData *rawWebFeaturesJSONData + featureData *rawWebFeaturesJSONDataV2 expectedValue *webdxfeaturetypes.ProcessedWebFeaturesData }{ { name: "catch-all case", - featureData: &rawWebFeaturesJSONData{ - Browsers: web_platform_dx__web_features.Browsers{ - Chrome: web_platform_dx__web_features.BrowserData{ - Name: "chrome", - Releases: nil, - }, - ChromeAndroid: web_platform_dx__web_features.BrowserData{ - Name: "chrome_android", - Releases: nil, - }, - Edge: web_platform_dx__web_features.BrowserData{ - Name: "edge", - Releases: nil, - }, - Firefox: web_platform_dx__web_features.BrowserData{ - Name: "firefox", - Releases: nil, - }, - FirefoxAndroid: web_platform_dx__web_features.BrowserData{ - Name: "firefox_android", - Releases: nil, - }, - Safari: web_platform_dx__web_features.BrowserData{ - Name: "safari", - Releases: nil, - }, - SafariIos: web_platform_dx__web_features.BrowserData{ - Name: "safari_ios", - Releases: nil, - }, - }, + featureData: &rawWebFeaturesJSONDataV2{ + Browsers: testBrowsers(), Groups: nil, Snapshots: nil, Features: map[string]web_platform_dx__web_features.FeatureValue{ @@ -315,3 +371,226 @@ func TestPostProcess(t *testing.T) { }) } } + +func TestPostProcessV3(t *testing.T) { + testCases := []struct { + name string + featureData *rawWebFeaturesJSONDataV3 + expectedValue *webdxfeaturetypes.ProcessedWebFeaturesData + expectedErr error + }{ + { + name: "catch-all case", + featureData: &rawWebFeaturesJSONDataV3{ + Browsers: testBrowsers(), + Groups: map[string]web_platform_dx__web_features.GroupData{ + "group1": { + Name: "Group 1", + Parent: nil, + }, + }, + Snapshots: map[string]web_platform_dx__web_features.SnapshotData{ + "snapshot1": { + Name: "Snapshot 1", + Spec: "spec1", + }, + }, + Features: json.RawMessage(` +{ + "feature1": { + "kind": "feature", + "compat_features": [ + "compat1", + "compat2" + ], + "description": "description", + "description_html": "description html", + "discouraged": { + "according_to": [ + "discouraged1", + "discouraged2" + ], + "alternatives": [ + "feature2", + "feature3" + ] + }, + "name": "feature 1 name", + "caniuse": "caniuse_data", + "group": [ + "group1", + "group2" + ], + "snapshot": [ + "snapshot1", + "snapshot2" + ], + "spec": [ + "spec1", + "spec2" + ], + "status": { + "baseline": "high", + "baseline_high_date": "≤2023-01-01", + "baseline_low_date": "≤2022-12-01", + "support": { + "chrome": "≤99", + "chrome_android": "≤98", + "firefox": "≤97", + "firefox_android": "≤96", + "edge": "≤95", + "safari": "≤94", + "safari_ios": "≤93" + } + } + }, + "feature2": { + "kind": "split", + "redirected_created_date": "2000-01-01", + "redirect_targets": [ + "feature1", + "feature3" + ] + }, + "feature3": { + "kind": "moved", + "redirect_target": "feature4", + "redirect_created_date": "2001-01-01" + } +}`), + }, + expectedValue: &webdxfeaturetypes.ProcessedWebFeaturesData{ + Browsers: web_platform_dx__web_features.Browsers{ + Chrome: web_platform_dx__web_features.BrowserData{ + Name: "chrome", + Releases: nil, + }, + ChromeAndroid: web_platform_dx__web_features.BrowserData{ + Name: "chrome_android", + Releases: nil, + }, + Edge: web_platform_dx__web_features.BrowserData{ + Name: "edge", + Releases: nil, + }, + Firefox: web_platform_dx__web_features.BrowserData{ + Name: "firefox", + Releases: nil, + }, + FirefoxAndroid: web_platform_dx__web_features.BrowserData{ + Name: "firefox_android", + Releases: nil, + }, + Safari: web_platform_dx__web_features.BrowserData{ + Name: "safari", + Releases: nil, + }, + SafariIos: web_platform_dx__web_features.BrowserData{ + Name: "safari_ios", + Releases: nil, + }, + }, + Features: &webdxfeaturetypes.FeatureKinds{ + Data: map[string]web_platform_dx__web_features.FeatureValue{ + "feature1": { + CompatFeatures: []string{"compat1", "compat2"}, + Description: "description", + DescriptionHTML: "description html", + Discouraged: &web_platform_dx__web_features.Discouraged{ + AccordingTo: []string{ + "discouraged1", + "discouraged2", + }, + Alternatives: []string{ + "feature2", + "feature3", + }, + }, + Name: "feature 1 name", + Caniuse: &web_platform_dx__web_features.StringOrStringArray{ + String: valuePtr("caniuse_data"), + StringArray: nil, + }, + Group: &web_platform_dx__web_features.StringOrStringArray{ + String: nil, + StringArray: []string{ + "group1", + "group2", + }, + }, + Snapshot: &web_platform_dx__web_features.StringOrStringArray{ + String: nil, + StringArray: []string{ + "snapshot1", + "snapshot2", + }, + }, + Spec: &web_platform_dx__web_features.StringOrStringArray{ + String: nil, + StringArray: []string{ + "spec1", + "spec2", + }, + }, + Status: web_platform_dx__web_features.Status{ + ByCompatKey: nil, + Baseline: &web_platform_dx__web_features.BaselineUnion{ + Bool: nil, + Enum: valuePtr(web_platform_dx__web_features.High), + }, + BaselineHighDate: valuePtr("2023-01-01"), + BaselineLowDate: valuePtr("2022-12-01"), + Support: web_platform_dx__web_features.StatusSupport{ + Chrome: valuePtr("99"), + ChromeAndroid: valuePtr("98"), + Firefox: valuePtr("97"), + FirefoxAndroid: valuePtr("96"), + Edge: valuePtr("95"), + Safari: valuePtr("94"), + SafariIos: valuePtr("93"), + }, + }, + }, + }, + Moved: map[string]web_platform_dx__web_features.FeatureMovedData{ + "feature3": { + Kind: "moved", + RedirectTarget: "feature4", + }, + }, + Split: map[string]web_platform_dx__web_features.FeatureSplitData{ + "feature2": { + Kind: "split", + RedirectTargets: []string{"feature1", "feature3"}, + }, + }, + }, + Groups: map[string]web_platform_dx__web_features.GroupData{ + "group1": { + Name: "Group 1", + Parent: nil, + }, + }, + Snapshots: map[string]web_platform_dx__web_features.SnapshotData{ + "snapshot1": { + Name: "Snapshot 1", + Spec: "spec1", + }, + }, + }, + expectedErr: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + value, err := postProcessV3(tc.featureData) + if diff := cmp.Diff(tc.expectedValue, value); diff != "" { + t.Errorf("postProcess unexpected output (-want +got):\n%s", diff) + } + if !errors.Is(err, tc.expectedErr) { + t.Errorf("postProcessV3 unexpected error expected %v received %v", tc.expectedErr, err) + } + }) + } +}