tokens ref from cucumber#377

Author: fperrad

testdata/good/datatables.feature.md.tokens

@@ -0,0 +1,9 @@
+(1:4)FeatureLine:()Feature/DataTables/
+(2:1)Empty://
+(3:5)ScenarioLine:()Scenario/minimalistic/
+(4:1)Empty://
+(5:3)StepLine:(Context)Given /a simple data table/
+(6:3)TableRow://5:foo,11:bar
+(7:1)Comment:/  | --- | --- |/
+(8:3)TableRow://5:boz,11:boo
+EOF

testdata/good/docstrings.feature.md.tokens

@@ -0,0 +1,9 @@
+(1:4)FeatureLine:()Feature/DocString variations/
+(2:1)Empty://
+(3:5)ScenarioLine:()Scenario/minimalistic/
+(4:1)Empty://
+(5:3)StepLine:(Conjunction)And /a DocString with an implicitly escaped separator inside/
+(6:1)DocStringSeparator:()````//
+(7:1)Other:/```/
+(8:1)DocStringSeparator:()````//
+EOF

testdata/good/minimal.feature.md.tokens

@@ -0,0 +1,6 @@
+(1:3)FeatureLine:()Feature/Minimal/
+(2:1)Empty://
+(3:4)ScenarioLine:()Scenario/minimalistic/
+(4:1)Empty://
+(5:6)StepLine:(Context)Given /the minimalism/
+EOF

testdata/good/misc.feature.md.tokens

@@ -0,0 +1,26 @@
+(1:1)FeatureLine:()Feature/Markdown document without "# Feature:" header/
+(2:1)Empty://
+(3:1)Empty://
+(4:1)Empty://
+(5:1)Empty://
+(6:1)Empty://
+(7:1)Empty://
+(8:1)Empty://
+(9:1)Empty://
+(10:1)Empty://
+(11:1)Empty://
+(12:3)ScenarioLine:()Scenario/Something about math/
+(13:3)StepLine:(Context)Given /step one/
+(14:3)StepLine:(Action)When /step two/
+(15:3)StepLine:(Outcome)Then /step three/
+(16:1)Empty://
+(17:3)ScenarioLine:()Scenario/Something about gravity/
+(18:4)StepLine:(Context)Given /step one/
+(19:4)StepLine:(Action)When /step two/
+(20:4)StepLine:(Outcome)Then /step three/
+(21:1)Empty://
+(22:1)Empty://
+(23:1)Empty://
+(24:1)Empty://
+(25:1)Empty://
+EOF

testdata/good/tags.feature.md.tokens

@@ -0,0 +1,43 @@
+(1:1)TagLine://2:@feature_tag1,18:@feature_tag2
+(2:3)TagLine://2:@feature_tag3
+(3:3)FeatureLine:()Feature/Minimal Scenario Outline/
+(4:1)Empty://
+(5:1)TagLine://2:@scenario_tag1,19:@scenario_tag2
+(6:3)TagLine://2:@scenario_tag3
+(7:4)ScenarioLine:()Scenario/minimalistic/
+(8:3)StepLine:(Context)Given /the minimalism/
+(9:1)Empty://
+(10:1)TagLine://2:@so_tag1,14:@so_tag2
+(11:3)TagLine://2:@so_tag3
+(12:4)ScenarioLine:()Scenario Outline/minimalistic outline/
+(13:3)StepLine:(Context)Given /the <what>/
+(14:1)Empty://
+(15:1)TagLine://2:@ex_tag1,13:@ex_tag2
+(16:3)TagLine://2:@ex_tag3
+(17:5)ExamplesLine:()Examples//
+(18:3)TableRow://5:what
+(19:3)Empty://
+(20:3)TableRow://5:minimalism
+(21:1)Empty://
+(22:1)TagLine://2:@ex_tag4,13:@ex_tag5
+(23:3)TagLine://2:@ex_tag6
+(24:5)ExamplesLine:()Examples//
+(25:3)TableRow://5:what
+(26:3)Empty://
+(27:3)TableRow://5:more minimalism
+(28:1)Empty://
+(29:1)TagLine://2:@comment_tag1
+(30:4)ScenarioLine:()Scenario/comments/
+(31:1)Other:/  Given a comment/
+(32:1)Other://
+(33:1)TagLine://2:@comment_tag#2
+(34:4)ScenarioLine:()Scenario/hash in tags/
+(35:1)Other:/  Given a comment is preceded by a space/
+(36:1)Other://
+(37:1)TagLine://2:@rule_tag
+(38:4)RuleLine:()Rule//
+(39:1)Empty://
+(40:1)TagLine://2:@joined_tag3,16:@joined_tag4
+(41:5)ScenarioLine:()Scenario/joined tags/
+(42:1)Other:/  Given the @delimits tags/
+EOF