[r] Fix: AnVIL uses uncommon encoding for MD5 digests (#7154)

Author: nadove-ucsc

src/azul/plugins/repository/tdr_anvil/__init__.py

@@ -351,7 +351,15 @@ def list_files(self, source: TDRSourceRef, prefix: str) -> list[AnvilFile]:
         batch = self._get_batch(source.spec,
                                 'anvil_file',
                                 prefix,
-                                key_column='file_md5sum')
+                                key_column=self._column_from_64_to_hex('file_md5sum'))
+
+        def missing_md5(row: BigQueryRow) -> bool:
+            missing = row['file_md5sum'] is None
+            if missing:
+                assert source.spec.name == 'ANVIL_1000G_2019_Dev_20230609_ANV5_202306121732', R(
+                    'File lacks MD5 digest', source, dict(row))
+            return missing
+
         return [
             AnvilFile(uuid=ref.entity_id,
                       name=row['file_name'],
@@ -360,6 +368,7 @@ def list_files(self, source: TDRSourceRef, prefix: str) -> list[AnvilFile]:
                       md5=row['file_md5sum'],
                       drs_uri=row['file_ref'])
             for ref, row in batch
+            if not missing_md5(row)
         ]
 
     def _emulate_bundle(self, bundle_fqid: TDRAnvilBundleFQID) -> TDRAnvilBundle:
@@ -953,4 +962,11 @@ def _columns(self, table_name: str) -> set[str]:
         else:
             columns = set(columns)
             columns.add('datarepo_row_id')
+            if table_name == 'anvil_file':
+                column = 'file_md5sum'
+                columns.remove(column)
+                columns.add(f'{self._column_from_64_to_hex(column)} AS {column}')
             return columns
+
+    def _column_from_64_to_hex(self, column: str) -> str:
+        return f'TO_HEX(FROM_BASE64({column}))'

test/indexer/data/595c469e-604d-ab34-af39-f5b9f5d61818.tdr.anvil.json

@@ -127,7 +127,7 @@
             "datarepo_row_id": "15b76f9c-6b46-433f-851d-34e89f1b9ba6",
             "file_format": ".vcf.gz",
             "file_id": "1e269f04-4347-4188-b060-1dcc69e71d67",
-            "file_md5sum": "vuxgbuCqKZ/fkT9CWTFmIg==",
+            "file_md5sum": "beec606ee0aa299fdf913f4259316622",
             "file_name": "307500.merged.matefixed.sorted.markeddups.recal.g.vcf.gz",
             "file_ref": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_1e269f04-4347-4188-b060-1dcc69e71d67",
             "file_size": 213021639,
@@ -149,7 +149,7 @@
             "datarepo_row_id": "3b17377b-16b1-431c-9967-e5d01fc5923f",
             "file_format": ".bam",
             "file_id": "8b722e88-8103-49c1-b351-e64fa7c6ab37",
-            "file_md5sum": "fNn9e1SovzgOROk3BvH6LQ==",
+            "file_md5sum": "7cd9fd7b54a8bf380e44e93706f1fa2d",
             "file_name": "307500.merged.matefixed.sorted.markeddups.recal.bam",
             "file_ref": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_8b722e88-8103-49c1-b351-e64fa7c6ab37",
             "file_size": 3306845592,
@@ -210,7 +210,7 @@
             "data_modality": [],
             "datarepo_row_id": "6b0f6c0f-5d80-4242-accb-840921351cd5",
             "file_format": ".txt",
-            "file_md5sum": "S/GBrRjzZAQYqh3rdiPYzA==",
+            "file_md5sum": "4bf181ad18f3640418aa1deb7623d8cc",
             "file_id": "1fab11f5-7eab-4318-9a58-68d8d06e0715",
             "file_name": "CCDG_13607_B01_GRM_WGS_2019-02-19_chr15.recalibrated_variants.annotated.coding.txt",
             "file_ref": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_1fab11f5-7eab-4318-9a58-68d8d06e0715",

test/indexer/data/826dea02-e274-affe-aabc-eb3db63ad068.results.json

@@ -294,7 +294,7 @@
             "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_1fab11f5-7eab-4318-9a58-68d8d06e0715",
             "file_format": ".txt",
             "file_id": "1fab11f5-7eab-4318-9a58-68d8d06e0715",
-            "file_md5sum": "S/GBrRjzZAQYqh3rdiPYzA==",
+            "file_md5sum": "4bf181ad18f3640418aa1deb7623d8cc",
             "file_name": "CCDG_13607_B01_GRM_WGS_2019-02-19_chr15.recalibrated_variants.annotated.coding.txt",
             "file_ref": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_1fab11f5-7eab-4318-9a58-68d8d06e0715",
             "file_size": 15079345,
@@ -419,7 +419,7 @@
             "drs_uri": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_8b722e88-8103-49c1-b351-e64fa7c6ab37",
             "file_format": ".bam",
             "file_id": "8b722e88-8103-49c1-b351-e64fa7c6ab37",
-            "file_md5sum": "fNn9e1SovzgOROk3BvH6LQ==",
+            "file_md5sum": "7cd9fd7b54a8bf380e44e93706f1fa2d",
             "file_name": "307500.merged.matefixed.sorted.markeddups.recal.bam",
             "file_ref": "drs://mock_tdr.lan/v1_6c87f0e1-509d-46a4-b845-7584df39263b_8b722e88-8103-49c1-b351-e64fa7c6ab37",
             "file_size": 3306845592,

test/indexer/data/826dea02-e274-affe-aabc-eb3db63ad068.tdr.anvil.json

@@ -1691,9 +1691,9 @@ def test_compact_manifest(self):
             ),
             (
                 'files.file_md5sum',
-                'S/GBrRjzZAQYqh3rdiPYzA==',
-                'vuxgbuCqKZ/fkT9CWTFmIg==',
-                'fNn9e1SovzgOROk3BvH6LQ=='
+                '4bf181ad18f3640418aa1deb7623d8cc',
+                'beec606ee0aa299fdf913f4259316622',
+                '7cd9fd7b54a8bf380e44e93706f1fa2d'
             ),
             (
                 'files.reference_assembly',