Release 1.15.1

Author: daviesrob

NEWS

@@ -1,3 +1,27 @@
+Noteworthy changes in release 1.15.1 (7th April 2022)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Security fix: Fixed broken error reporting in the sam_cap_mapq()
+  function, due to a missing hts_log() parameter.  Prior to this fix
+  it was possible to abuse the log message format string by passing
+  a specially crafted alignment record to this function. (PR#1406)
+
+* HTSlib now uses libhtscodecs release 1.2.2.  This fixes a number
+  of bugs where invalid compressed data could trigger usage of
+  uninitialised values. (PR#1416)
+
+* Fixed excessive memory used by multi-threaded SAM output on
+  long reads. (Part of PR#1384)
+
+* Fixed a bug where tabix would misinterpret region specifiers
+  starting at position 0.  It will also now warn if the file
+  being indexed is supposed to be 1-based but has positions
+  less than or equal to 0. (PR#1411)
+
+* The VCF header parser will now issue a warning if it finds an
+  INFO header with Type=Flag but Number not equal to 0.  It will
+  also ignore the incorrect Number so the flag can be used. (PR#1415)
+
 Noteworthy changes in release 1.15 (21st February 2022)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

bgzip.1

@@ -1,4 +1,4 @@
-.TH bgzip 1 "21 February 2022" "htslib-1.15" "Bioinformatics tools"
+.TH bgzip 1 "7 April 2022" "htslib-1.15.1" "Bioinformatics tools"
 .SH NAME
 .PP
 bgzip \- Block compression/decompression utility

hts.c

@@ -3711,14 +3711,17 @@ const char *hts_parse_region(const char *s, int *tid, hts_pos_t *beg,
     char *hyphen;
     *beg = hts_parse_decimal(colon+1, &hyphen, flags) - 1;
     if (*beg < 0) {
+        if (*beg != -1 && *hyphen == '-' && colon[1] != '\0') {
+            // User specified zero, but we're 1-based.
+            hts_log_error("Coordinates must be > 0");
+            return NULL;
+        }
         if (isdigit_c(*hyphen) || *hyphen == '\0' || *hyphen == ',') {
             // interpret chr:-100 as chr:1-100
             *end = *beg==-1 ? HTS_POS_MAX : -(*beg+1);
             *beg = 0;
             return s_end;
-        } else if (*hyphen == '-') {
-            *beg = 0;
-        } else {
+        } else if (*beg < -1) {
             hts_log_error("Unexpected string \"%s\" after region", hyphen);
             return NULL;
         }

htscodecs

@@ -1,4 +1,4 @@
-.TH htsfile 1 "21 February 2022" "htslib-1.15" "Bioinformatics tools"
+.TH htsfile 1 "7 April 2022" "htslib-1.15.1" "Bioinformatics tools"
 .SH NAME
 htsfile \- identify high-throughput sequencing data files
 .\"

htsfile.1

@@ -1,4 +1,4 @@
-.TH htslib-s3-plugin 7 "21 February 2022" "htslib-1.15" "Bioinformatics tools"
+.TH htslib-s3-plugin 7 "7 April 2022" "htslib-1.15.1" "Bioinformatics tools"
 .SH NAME
 s3 plugin \- htslib AWS S3 plugin
 .\"

htslib-s3-plugin.7

@@ -486,7 +486,7 @@ const char *hts_version(void);
 // Immediately after release, bump ZZ to 90 to distinguish in-development
 // Git repository builds from the release; you may wish to increment this
 // further when significant features are merged.
-#define HTS_VERSION 101500
+#define HTS_VERSION 101501
 
 /*! @abstract Introspection on the features enabled in htslib
  *

htslib/hts.h

@@ -91,12 +91,12 @@ int sam_cap_mapq(bam1_t *b, const char *ref, hts_pos_t ref_len, int thres)
 static int realn_check_tag(const uint8_t *tg, enum htsLogLevel severity,
                            const char *type, const bam1_t *b) {
     if (*tg != 'Z') {
-        hts_log(severity, "Incorrect %s tag type (%c) for read %s",
+        hts_log(severity, __func__, "Incorrect %s tag type (%c) for read %s",
                 type, *tg, bam_get_qname(b));
         return -1;
     }
     if (b->core.l_qseq != strlen((const char *) tg + 1)) {
-        hts_log(severity, "Read %s %s tag is wrong length",
+        hts_log(severity, __func__, "Read %s %s tag is wrong length",
                 bam_get_qname(b), type);
         return -1;
     }

realn.c

@@ -2906,9 +2906,10 @@ ssize_t bam_parse_cigar(const char *in, char **end, bam1_t *b) {
  * SAM threading
  */
 // Size of SAM text block (reading)
-#define NM 240000
-// Number of BAM records (writing)
-#define NB 1000
+#define SAM_NBYTES 240000
+
+// Number of BAM records (writing, up to NB_mem in size)
+#define SAM_NBAM 1000
 
 struct SAM_state;
 
@@ -2918,7 +2919,8 @@ typedef struct sp_bams {
     int serial;
 
     bam1_t *bams;
-    int nbams, abams; // used and alloc
+    int nbams, abams; // used and alloc for bams[] array
+    size_t bam_mem;   // very approximate total size
 
     struct SAM_state *fd;
 } sp_bams;
@@ -3169,6 +3171,7 @@ static void *sam_parse_worker(void *arg) {
             goto err;
         }
         gb->nbams = 0;
+        gb->bam_mem = 0;
     }
     gb->serial = gl->serial;
     gb->next = NULL;
@@ -3221,6 +3224,7 @@ static void *sam_parse_worker(void *arg) {
             cleanup_sp_lines(gl);
             goto err;
         }
+
         cp = nl;
         i++;
     }
@@ -3290,7 +3294,7 @@ static void *sam_dispatcher_read(void *vp) {
             l = calloc(1, sizeof(*l));
             if (!l)
                 goto err;
-            l->alloc = NM;
+            l->alloc = SAM_NBYTES;
             l->data = malloc(l->alloc+8); // +8 for optimisation in sam_parse1
             if (!l->data) {
                 free(l);
@@ -3301,11 +3305,11 @@ static void *sam_dispatcher_read(void *vp) {
         }
         l->next = NULL;
 
-        if (l->alloc < line_frag+NM/2) {
-            char *rp = realloc(l->data, line_frag+NM/2 +8);
+        if (l->alloc < line_frag+SAM_NBYTES/2) {
+            char *rp = realloc(l->data, line_frag+SAM_NBYTES/2 +8);
             if (!rp)
                 goto err;
-            l->alloc = line_frag+NM/2;
+            l->alloc = line_frag+SAM_NBYTES/2;
             l->data = rp;
         }
         memcpy(l->data, line.s, line_frag);
@@ -4302,16 +4306,18 @@ int sam_write1(htsFile *fp, const sam_hdr_t *h, const bam1_t *b)
                     fd->bams = gb->next;
                     gb->next = NULL;
                     gb->nbams = 0;
+                    gb->bam_mem = 0;
                     pthread_mutex_unlock(&fd->lines_m);
                 } else {
                     pthread_mutex_unlock(&fd->lines_m);
                     if (!(gb = calloc(1, sizeof(*gb)))) return -1;
-                    if (!(gb->bams = calloc(NB, sizeof(*gb->bams)))) {
+                    if (!(gb->bams = calloc(SAM_NBAM, sizeof(*gb->bams)))) {
                         free(gb);
                         return -1;
                     }
                     gb->nbams = 0;
-                    gb->abams = NB;
+                    gb->abams = SAM_NBAM;
+                    gb->bam_mem = 0;
                     gb->fd = fd;
                     fd->curr_idx = 0;
                     fd->curr_bam = gb;
@@ -4320,11 +4326,11 @@ int sam_write1(htsFile *fp, const sam_hdr_t *h, const bam1_t *b)
 
             if (!bam_copy1(&gb->bams[gb->nbams++], b))
                 return -2;
+            gb->bam_mem += b->l_data + sizeof(*b);
 
             // Dispatch if full
-            if (gb->nbams == NB) {
+            if (gb->nbams == SAM_NBAM || gb->bam_mem > SAM_NBYTES*0.8) {
                 gb->serial = fd->serial++;
-                //fprintf(stderr, "Dispatch another %d bams\n", NB);
                 pthread_mutex_lock(&fd->command_m);
                 if (fd->errcode != 0) {
                     pthread_mutex_unlock(&fd->command_m);

sam.c

@@ -1,4 +1,4 @@
-.TH tabix 1 "21 February 2022" "htslib-1.15" "Bioinformatics tools"
+.TH tabix 1 "7 April 2022" "htslib-1.15.1" "Bioinformatics tools"
 .SH NAME
 .PP
 tabix \- Generic indexer for TAB-delimited genome position files
@@ -81,8 +81,8 @@ greater than that, you will need to use a CSI index.
 .SH INDEXING OPTIONS
 .TP 10
 .B -0, --zero-based
-Specify that the position in the data file is 0-based (e.g. UCSC files)
-rather than 1-based.
+Specify that the position in the data file is 0-based half-open
+(e.g. UCSC files) rather than 1-based.
 .TP
 .BI "-b, --begin " INT
 Column of start chromosomal position. [4]