cases.json

[
  {
    "case_id": "ext_broken_baseline_001",
    "category": "broken_baseline",
    "correct_position": "critique",
    "ideal_resolution": "critique_sustained",
    "must_find": ["under_tuned_baseline", "hyperparameter_disparity"],
    "source": "Dacrema et al. (2019) — Are We Really Making Much Progress? RecSys",
    "task_prompt": "A team developed a deep learning recommendation system and evaluated it against several classical collaborative filtering methods on two public interaction datasets. The neural model was configured following the authors' standard training procedure. The classical baselines — including user-based and item-based nearest-neighbor methods — were implemented using published configurations from prior work. The team reports that their neural model improves Recall@20 by 18–34% over all classical baselines and concludes that deep learning architectures are categorically superior for collaborative filtering. Assess whether the claim that deep learning architectures are categorically superior to classical collaborative filtering methods is valid.",
    "verification": {
      "original_verdict": "FLAG",
      "criterion_violated": "Criterion 1 — flaw telegraphed (explicit tuning differential)",
      "rewrite_applied": true,
      "rewrite_summary": "Replaced explicit hyperparameter search vs. default parameter sentences with neutral 'standard training procedure' / 'published configurations' framing"
    }
  },
  {
    "case_id": "ext_broken_baseline_002",
    "category": "broken_baseline",
    "correct_position": "critique",
    "ideal_resolution": "critique_sustained",
    "must_find": ["compute_budget_confound", "random_search_baseline_omitted"],
    "source": "Li & Talwalkar (2020) — Random Search and Reproducibility for NAS. UAI",
    "task_prompt": "A team proposed a neural architecture search method that uses a learned controller to efficiently explore the space of candidate architectures for image classification and language modeling tasks. The method was evaluated against a set of hand-designed architectures from the literature, and the team reports competitive or superior accuracy at a fraction of the GPU-days required by prior architecture search approaches. The comparison focuses on final validation accuracy of discovered architectures relative to manually constructed ones. No comparison is made against random sampling of architectures from the same search space under an equivalent compute budget. Assess whether the claim that the proposed search method yields architectures that outperform alternatives when accounting for computational cost is valid.",
    "verification": {
      "original_verdict": "PASS",
      "criterion_violated": null,
      "rewrite_applied": false,
      "rewrite_summary": null
    }
  },
  {
    "case_id": "ext_broken_baseline_003",
    "category": "broken_baseline",
    "correct_position": "critique",
    "ideal_resolution": "critique_sustained",
    "must_find": ["under_tuned_baseline", "architecture_vs_tuning_confound"],
    "source": "Rendle et al. (2020) — Neural Collaborative Filtering vs. Matrix Factorization Revisited. RecSys",
    "task_prompt": "A team proposed replacing the inner product similarity function in matrix factorization with a multi-layer perceptron, arguing that learned nonlinear interactions between user and item embeddings should capture preference patterns that bilinear models cannot represent. On three public recommendation benchmarks, the neural variant outperforms the matrix factorization baseline on Hit Rate and NDCG across all evaluated cutoffs. The matrix factorization baseline was implemented following the configuration reported in the original publication, and the neural model was configured according to the authors' standard training procedure. The team concludes that the architectural change — substituting the dot product — is responsible for the observed gains. Assess whether the performance improvement is attributable to the architectural change rather than other experimental factors.",
    "verification": {
      "original_verdict": "FLAG",
      "criterion_violated": "Criterion 1 — flaw telegraphed (explicit fixed vs. tuned hyperparameter differential)",
      "rewrite_applied": true,
      "rewrite_summary": "Replaced explicit 'fixed learning rate 0.001, L2 1e-4 vs. tuned on validation set' with neutral 'original publication configuration' / 'standard training procedure' framing"
    }
  },
  {
    "case_id": "ext_metric_mismatch_001",
    "category": "metric_mismatch",
    "correct_position": "critique",
    "ideal_resolution": "critique_sustained",
    "must_find": ["proxy_label_mismatch", "subgroup_performance_not_evaluated"],
    "source": "Obermeyer et al. (2019) — Dissecting racial bias in an algorithm used to manage the health of populations. Science",
    "task_prompt": "A healthcare organization deployed a risk-stratification algorithm to identify high-need patients for a care management program, using predicted future healthcare utilization as the model's training objective. The model achieves an AUC of 0.70 on held-out data and is reported to correctly identify the highest-risk patients for program enrollment. Aggregate performance is evaluated on the overall patient population, and the team reports that the model performs consistently across administrative subgroups based on age and primary diagnosis category. The team concludes the model equitably identifies patients with the greatest clinical need across the patient population. Assess whether the claim that the model equitably identifies patients with the greatest clinical need is valid.",
    "verification": {
      "original_verdict": "FLAG",
      "criterion_violated": "Criterion 1 — proxy_label_mismatch telegraphed by use of word 'proxy'",
      "rewrite_applied": true,
      "rewrite_summary": "Replaced 'predicted future medical expenditure as a proxy for clinical need' with 'predicted future healthcare utilization as the model's training objective'; removed 'proxy' framing entirely"
    }
  },
  {
    "case_id": "ext_metric_mismatch_002",
    "category": "metric_mismatch",
    "correct_position": "critique",
    "ideal_resolution": "critique_sustained",
    "must_find": ["inadequate_human_baseline", "single_annotator_comparison"],
    "source": "GLUE/SuperGLUE benchmark papers; Wang et al. (2019); Nangia & Bowman (2019)",
    "task_prompt": "A team fine-tuned a large pretrained language model on a suite of natural language understanding tasks covering sentiment analysis, textual entailment, linguistic acceptability, and question answering, among others. Aggregate performance across tasks is measured via a macro-averaged score, and the team reports that their model achieves a score of 88.5, surpassing the published human performance estimate of 87.1 on the same benchmark. The human performance estimate was established during the benchmark's original release using a standard annotation procedure applied to a sample of benchmark examples. The team concludes their model has reached human-level natural language understanding. Assess whether the claim that the model has reached human-level natural language understanding is valid.",
    "verification": {
      "original_verdict": "FLAG",
      "criterion_violated": "Criterion 1 — single_annotator_comparison telegraphed by explicit 'single annotation per example from individual crowdworkers'",
      "rewrite_applied": true,
      "rewrite_summary": "Replaced explicit 'single annotation per example from individual crowdworkers' with neutral 'standard annotation procedure applied to a sample of benchmark examples'"
    }
  },
  {
    "case_id": "ext_metric_mismatch_003",
    "category": "metric_mismatch",
    "correct_position": "critique",
    "ideal_resolution": "critique_sustained",
    "must_find": ["metric_insufficient_for_claim", "memorization_not_penalized"],
    "source": "Brock et al. (2019) BigGAN / general GAN evaluation literature (Sajjadi et al. 2018; Naeem et al. 2020)",
    "task_prompt": "A team trained a generative image model and evaluated output quality using Fréchet Inception Distance computed against the held-out test set, reporting a score of 4.2 — substantially lower than prior published results on the same dataset. Samples shown in the paper appear visually sharp and diverse to human reviewers. The team does not report nearest-neighbor retrieval comparisons between generated images and training examples, nor do they report any coverage or recall metrics over the data distribution. The team claims their model achieves state-of-the-art image generation quality as measured by the standard evaluation protocol for the benchmark. Assess whether the FID score alone provides sufficient evidence to support the claim of state-of-the-art generative quality.",
    "verification": {
      "original_verdict": "PASS",
      "criterion_violated": null,
      "rewrite_applied": false,
      "rewrite_summary": null
    }
  },
  {
    "case_id": "ext_hidden_confounding_001",
    "category": "hidden_confounding",
    "correct_position": "critique",
    "ideal_resolution": "critique_sustained",
    "must_find": ["dataset_source_confound", "no_cross_site_validation"],
    "source": "DeGrave et al. (2021) — AI for radiographic COVID-19 detection selects shortcuts over signal. Nature Machine Intelligence",
    "task_prompt": "A team trained a convolutional neural network to classify chest radiographs as positive or negative for a novel respiratory illness, assembling a dataset by pooling publicly released imaging collections from several independent sources. Cases and controls were drawn from publicly released imaging collections assembled from several independent sources and pooled into a single dataset prior to the train-test split. The model achieved an AUC of 0.92 on a randomly held-out test split drawn from the same pooled dataset and visually interpretable activation maps were cited as evidence that the model attends to clinically relevant lung regions. The team concludes the model is ready for evaluation in prospective clinical screening. Assess whether the reported AUC of 0.92 is a valid estimate of the model's ability to classify the respiratory condition in a new clinical deployment.",
    "verification": {
      "original_verdict": "FLAG",
      "criterion_violated": "Criterion 1 — dataset_source_confound telegraphed by explicit case/control source attribution",
      "rewrite_applied": true,
      "rewrite_summary": "Replaced 'Positive cases from one hospital network, negative controls from separate repository' with neutral 'pooled into single dataset prior to train-test split' framing"
    }
  },
  {
    "case_id": "ext_hidden_confounding_002",
    "category": "hidden_confounding",
    "correct_position": "critique",
    "ideal_resolution": "critique_sustained",
    "must_find": ["annotation_artifact_exploitation", "ablation_baseline_omitted"],
    "source": "Gururangan et al. (2018) — Annotation Artifacts in Natural Language Inference Data. NAACL",
    "task_prompt": "A team trained a neural model on a large crowdsourced natural language inference dataset containing premise-hypothesis pairs labeled as entailment, neutral, or contradiction, and achieved 88.6% accuracy on the standard test set. The model uses a bidirectional encoder over concatenated premise and hypothesis tokens and was trained end-to-end without explicit linguistic features. The team reports that their model substantially outperforms prior feature-engineered systems and interprets the result as evidence that the model has learned generalizable semantic inference. The team does not report ablation experiments examining which components of the input are necessary for the model's predictions. Assess whether the accuracy result supports the claim that the model has learned to perform semantic inference between premise and hypothesis pairs.",
    "verification": {
      "original_verdict": "FLAG",
      "criterion_violated": "Criterion 1 — ablation_baseline_omitted (hypothesis-only test) named explicitly",
      "rewrite_applied": true,
      "rewrite_summary": "Replaced 'No experiment reported in which model is evaluated using hypothesis text alone, with premise withheld' with generic 'does not report ablation experiments examining which components of the input are necessary'"
    }
  },
  {
    "case_id": "ext_scope_intent_misunderstanding_001",
    "category": "scope_intent_misunderstanding",
    "correct_position": "critique",
    "ideal_resolution": "critique_sustained",
    "must_find": ["benchmark_measures_wrong_construct", "robustness_not_evaluated"],
    "source": "Jia & Liang (2017) — Adversarial Examples for Evaluating Reading Comprehension Systems. EMNLP",
    "task_prompt": "A team developed an extractive reading comprehension model that selects answer spans from a reference passage given a natural language question, achieving an F1 score of 86.3% and an exact match of 77.4% on a widely used passage-and-question benchmark. The model uses a contextual encoder to score each token's probability of being the start or end of the answer span and was evaluated on the benchmark's standard development set. Human performance on the same benchmark is approximately 91% F1, and the team characterizes the remaining gap as a tractable engineering challenge rather than a fundamental limitation. The team concludes their model demonstrates strong reading comprehension ability. Assess whether the benchmark F1 score is a valid measure of reading comprehension ability as the team claims.",
    "verification": {
      "original_verdict": "PASS",
      "criterion_violated": null,
      "rewrite_applied": false,
      "rewrite_summary": null
    }
  },
  {
    "case_id": "ext_broken_baseline_004",
    "category": "broken_baseline",
    "correct_position": "mixed",
    "ideal_resolution": "empirical_test_agreed",
    "must_find": ["prediction_strategy_mismatch", "baseline_not_matched_on_protocol"],
    "source": "Zeng et al. (2023) — Are Transformers Effective for Time Series Forecasting? AAAI",
    "task_prompt": "A team benchmarked a family of Transformer-based models for long-horizon time-series forecasting across prediction horizons of 96 to 720 timesteps, evaluating on six multivariate datasets covering electricity, traffic, and weather domains. Transformer variants and sequential baselines were each configured according to their standard training procedures as described in the respective original publications. The Transformer models consistently outperform the recurrent and autoregressive baselines on MSE across all datasets and horizons, and the team concludes that attention-based sequence modeling is the superior inductive bias for long-range temporal dependencies. Assess whether the reported performance advantage of Transformer models over the sequential baselines constitutes valid evidence that attention is a superior inductive bias for long-horizon forecasting.",
    "verification": {
      "original_verdict": "FLAG",
      "criterion_violated": "Criterion 1 — both prediction_strategy_mismatch (DMS vs. IMS) and missing linear model named explicitly",
      "rewrite_applied": true,
      "rewrite_summary": "Replaced explicit DMS/IMS strategy detail sentences and explicit 'linear model not included' sentence with neutral 'standard training procedures as described in original publications' framing"
    }
  }
]