add first insd_curation package (#103)

* add first insd_curation package

* code cleanup

* fix possible external data consumption fail

Author: kMutagene

StagingArea/insdc_curation/insdc_curation@0.0.1.fsx

@@ -0,0 +1,169 @@
+let [<Literal>]PACKAGE_METADATA = """(*
+---
+Name: insdc_curation
+Description: Validates certain INSDC record curation metadata.
+Summary: |
+  Validates certain INSDC record curation metadata.
+    - fastqc files
+      - experiment accession annotation is present and correct
+      - descends from a correctly named sample
+MajorVersion: 0
+MinorVersion: 0
+PatchVersion: 1
+Publish: true
+Authors:
+  - FullName: Kevin Schneider
+    Affiliation: DataPLANT
+Tags:
+  - Name: INSDC
+  - Name: Metadata curation
+ReleaseNotes: |
+  - fastqc process graph validation
+---
+*)"""
+
+#r "nuget: ARCtrl"
+#r "nuget: ARCtrl.QueryModel, 3.0.0-alpha.4"
+#r "nuget: ARCExpect.Core, 7.0.0-alpha"
+#r "nuget: FsHttp"
+#r "nuget: FSharpAux"
+#r "nuget: FSharpAux.IO"
+
+open ARCtrl
+open ARCtrl.QueryModel
+open ARCExpect
+open FsHttp
+open FSharpAux.IO.SchemaReader
+open Expecto
+open System.IO
+open System
+
+let arcDir = Directory.GetCurrentDirectory()
+
+let arc =
+    try ARC.load arcDir with
+    | _ -> ARC(identifier = "unable to load arc from this dir")
+
+let project_accession = arc.Identifier
+
+module Domain =
+
+    open FSharpAux.IO.SchemaReader.Attribute
+    open System.Collections.Generic
+
+    /// represents the relations provided by ENA portal API in TSV form
+    type INSDC_Relations = {
+        [<FieldAttribute("study_accession")>]
+        study_accession: string
+        [<FieldAttribute("sample_accession")>]
+        sample_accession: string
+        [<FieldAttribute("experiment_accession")>]
+        experiment_accession: string
+        [<FieldAttribute("run_accession")>]
+        run_accession: string
+        [<FieldAttribute("fastq_ftp")>]
+        fastq_ftp: string
+    }
+
+    /// Some records map from sample to multiple fastq files in a single line (e.g., paired end reads). This function splits such records into multiple records, one for each fastq file.
+    let splitByFastq (r: INSDC_Relations) =
+        match r.fastq_ftp.Split(';') |> Array.filter (fun s -> s <> "") with
+        | [||]  -> [ r ]                                        // no fastq → keep row as-is
+        | files -> [ for f in files -> { r with fastq_ftp = f } ]
+
+    /// lookup index for retrieving INSDC_Relations by any of the accession numbers or fastq file name. Last-wins in case of duplicates.
+    let buildIndex (records: INSDC_Relations seq) =
+        let d = Dictionary<string, INSDC_Relations>(StringComparer.OrdinalIgnoreCase)
+        for r in records do
+            for v in [ 
+                r.study_accession
+                r.sample_accession
+                r.experiment_accession
+                r.run_accession
+                r.fastq_ftp 
+            ] do
+                if not (String.IsNullOrEmpty v) then d[v] <- r   // last-wins
+        d
+
+    module OntologyTerms = 
+        let experiment_accession_term =
+            OntologyAnnotation.fromTermAnnotation(
+                "NCIT:C175892",
+                name = "Experiment Accession Number"
+            )
+
+module ExpectedData =
+
+    let reader = new Csv.CsvReader<Domain.INSDC_Relations>(SchemaMode=Csv.Fill)
+
+    let relations = 
+        try
+            http {
+                GET $"https://www.ebi.ac.uk/ena/portal/api/filereport?accession={project_accession}&result=read_run&fields=study_accession,sample_accession,experiment_accession,run_accession,tax_id,scientific_name,fastq_ftp,submitted_ftp,bam_ftp&format=tsv&download=true&limit=0"
+            }
+            |> Request.send
+            |> Response.toText
+            |> fun response -> reader.ReadFromString(response, '\t', firstLineHasHeader = true)
+            |> Seq.collect Domain.splitByFastq
+            |> List.ofSeq
+        with
+            | _ -> []
+
+    let relations_index = Domain.buildIndex relations
+
+    let tryFindRelation (s: string) =
+        match relations_index.TryGetValue s with
+        | true, r  -> Some r
+        | false, _ -> None
+
+
+let create_insdc_relation_validation_cases_for_fastq_file (node: QNode) =
+
+    let samples =
+        arc.ArcTables.SamplesOf(node)
+        |> Seq.map (fun n -> n.Name)
+        |> Array.ofSeq
+
+    let experiment_accession_actual =
+        try 
+            Some (arc.PreviousParametersOf(node)).[Domain.OntologyTerms.experiment_accession_term]
+        with
+            | _ -> None
+
+    let expected_relations = Expect.wantSome (ExpectedData.tryFindRelation node.Name) "Experiment accession annotation is missing in INSDC relations" 
+
+    testList node.Name [
+
+        test $"has correct experiment accession annotation (expected: {expected_relations.experiment_accession})" {
+            let actual = Expect.wantSome experiment_accession_actual "Experiment accession annotation is missing"
+            Expect.equal actual.ValueText expected_relations.experiment_accession "Experiment accession annotation does not match node name"
+        }
+
+        test $"descends from correct sample (expected: {expected_relations.sample_accession})" {
+            Expect.contains samples expected_relations.sample_accession $"Sample accession {expected_relations.sample_accession} not found in ARC"
+        }
+    ]
+
+let fastqc_cases =
+    testList "fastqc" (
+        arc.TryGetAssay("sequencing")
+        |> Option.map (fun assay ->
+            assay.LastData
+            |> Seq.map (fun node -> create_insdc_relation_validation_cases_for_fastq_file node)
+            |> Seq.toList
+        ) 
+        |> Option.defaultValue []
+    )
+
+Setup.ValidationPackage(
+    metadata = Setup.Metadata(PACKAGE_METADATA, AVPRIndex.Frontmatter.FSharpFrontmatter),
+    CriticalValidationCases = [
+        test "has sequencing assay" {
+            Expect.isSome (arc.TryGetAssay("sequencing")) "No sequencing assay found"
+        }
+        fastqc_cases
+    ]
+)
+|> Execute.ValidationPipeline(
+    basePath = arcDir
+)