Skip to content

Commit 21d4bab

Browse files
eereiterzimzoom
eereiter
and
zimzoom
authored
CMR-8506 Adding Bulk granule update for ECHO10 OnlineAccessURLs (#1843)
* CMR-8506 Adding Bulk granule update for ECHO10 OnlineAccessURLs * Update ingest-app/resources/granule_bulk_update_schema.json Co-authored-by: K. Lucia Z. <7947215+zimzoom@users.noreply.github.com> * Update ingest-app/resources/granule_bulk_update_schema.json Co-authored-by: K. Lucia Z. <7947215+zimzoom@users.noreply.github.com> * Updated to fix comment. --------- Co-authored-by: K. Lucia Z. <7947215+zimzoom@users.noreply.github.com>
1 parent 86a118b commit 21d4bab

6 files changed

Lines changed: 238 additions & 360 deletions

File tree

‎ingest-app/resources/granule_bulk_update_schema.json‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
"Checksum",
1616
"Format",
1717
"MimeType",
18+
"OnlineAccessURL",
1819
"OnlineResourceURL",
1920
"OPeNDAPLink",
2021
"S3Link",
@@ -49,6 +50,24 @@
4950
}
5051
}
5152
},
53+
"UpdateURLType": {
54+
"type": "object",
55+
"additionalProperties": false,
56+
"description": "Describes an existing URL and replacement URL.",
57+
"properties": {
58+
"from": {
59+
"description": "The existing URL to be updated.",
60+
"type": "string",
61+
"minLength": 1
62+
},
63+
"to": {
64+
"description": "The new URL to be substituted",
65+
"type": "string",
66+
"minLength": 1
67+
}
68+
},
69+
"required": [ "from", "to" ]
70+
},
5271
"FileType": {
5372
"type": "object",
5473
"additionalProperties": false,
@@ -82,23 +101,11 @@
82101
"MimeType": {
83102
"type": "string"
84103
},
104+
"OnlineAccessURL": {
105+
"$ref": "#/definitions/UpdateURLType"
106+
},
85107
"OnlineResourceURL": {
86-
"type": "object",
87-
"additionalProperties": false,
88-
"description": "Describes an OnlineResource existing URL and replacment URL.",
89-
"properties": {
90-
"from": {
91-
"description": "The existing URL to updated.",
92-
"type": "string",
93-
"minLength": 1
94-
},
95-
"to": {
96-
"description": "The new URL to be substituted",
97-
"type": "string",
98-
"minLength": 1
99-
}
100-
},
101-
"required": [ "from", "to" ]
108+
"$ref": "#/definitions/UpdateURLType"
102109
},
103110
"Checksum": {
104111
"type": "object",

‎ingest-app/src/cmr/ingest/services/granule_bulk_update/mime_type/echo10.clj‎

Lines changed: 3 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -5,67 +5,10 @@
55
[clojure.data.xml :as xml]
66
[clojure.set :as set]
77
[clojure.string :as string]
8-
[clojure.zip :as zip]
98
[cmr.common.services.errors :as errors]
109
[cmr.common.xml :as cx]
1110
[cmr.ingest.services.granule-bulk-update.utils.echo10 :as echo10-utils]))
1211

13-
(defn- xml-elem->online-access-url
14-
"Parses and returns XML element for OnlineAccessURL"
15-
[elem]
16-
(let [url (cx/string-at-path elem [:URL])
17-
description (cx/string-at-path elem [:URLDescription])
18-
mime-type (cx/string-at-path elem [:MimeType])]
19-
{:url url
20-
:url-description description
21-
:mime-type mime-type}))
22-
23-
(defn- update-accesses
24-
"Constructs the new OnlineAccessURLs node in zipper representation"
25-
[online-access-urls url-map]
26-
(let [edn-access-urls (map xml-elem->online-access-url online-access-urls)
27-
access-urls (map #(merge %
28-
(when-let [mime-type (get url-map (:url %))]
29-
{:mime-type mime-type}))
30-
edn-access-urls)]
31-
(xml/element
32-
:OnlineAccessURLs {}
33-
(for [r access-urls]
34-
(let [{:keys [url url-description mime-type]} r]
35-
(xml/element :OnlineAccessURL {}
36-
(xml/element :URL {} url)
37-
(when url-description (xml/element :URLDescription {} url-description))
38-
(when mime-type (xml/element :MimeType {} mime-type))))))))
39-
40-
(defn- replace-in-tree
41-
"Take a parsed granule xml, replace the given node with the provided replacement
42-
Returns the zipper representation of the updated xml."
43-
[tree element-tag replacement]
44-
(let [zipper (zip/xml-zip tree)
45-
start-loc (zip/down zipper)]
46-
(loop [loc start-loc done false]
47-
(if done
48-
(zip/root loc)
49-
(if-let [right-loc (zip/right loc)]
50-
(cond
51-
;; at an OnlineResources element, replace the node with updated value
52-
(= element-tag (-> right-loc zip/node :tag))
53-
(recur (zip/replace right-loc replacement) true)
54-
55-
;; no action needs to be taken, move to the next node
56-
:else
57-
(recur right-loc false))
58-
(recur loc true))))))
59-
60-
(defn- update-online-access-urls
61-
"Return an OnlineAccessURLs node where MimeType is updated where the URL has a matching entry in the url-map."
62-
[tree url-map]
63-
(let [online-access-urls (cx/elements-at-path
64-
tree
65-
[:OnlineAccessURLs :OnlineAccessURL])]
66-
(when (seq online-access-urls)
67-
(update-accesses online-access-urls url-map))))
68-
6912
(defn update-mime-type
7013
"Update the the MimeType for elements within OnlineResources and OnlineAccess in echo10
7114
granule metadata and return granule."
@@ -93,10 +36,10 @@
9336
"]")]))
9437

9538
online-resources (echo10-utils/update-online-resources parsed :url :mime-type url-map)
96-
access-urls (update-online-access-urls parsed url-map)
39+
access-urls (echo10-utils/update-online-access-urls parsed :url :mime-type url-map)
9740

9841
updated-metadata (-> parsed
99-
(replace-in-tree :OnlineResources online-resources)
100-
(replace-in-tree :OnlineAccessURLs access-urls)
42+
(echo10-utils/replace-in-tree :OnlineResources online-resources)
43+
(echo10-utils/replace-in-tree :OnlineAccessURLs access-urls)
10144
xml/indent-str)]
10245
(assoc concept :metadata updated-metadata)))

‎ingest-app/src/cmr/ingest/services/granule_bulk_update/online_resource_url/echo10.clj‎

Lines changed: 30 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,32 +5,10 @@
55
[clojure.data.xml :as xml]
66
[clojure.set :as set]
77
[clojure.string :as string]
8-
[clojure.zip :as zip]
9-
[cmr.common.log :refer [info debug]]
108
[cmr.common.services.errors :as errors]
119
[cmr.common.xml :as cx]
1210
[cmr.ingest.services.granule-bulk-update.utils.echo10 :as echo10-utils]))
1311

14-
(defn- replace-in-tree
15-
"Take a parsed granule xml, replace the given node with the provided replacement
16-
Returns the zipper representation of the updated xml."
17-
[tree element-tag replacement]
18-
(let [zipper (zip/xml-zip tree)
19-
start-loc (zip/down zipper)]
20-
(loop [loc start-loc done false]
21-
(if done
22-
(zip/root loc)
23-
(if-let [right-loc (zip/right loc)]
24-
(cond
25-
;; at an OnlineResources element, replace the node with updated value
26-
(= element-tag (-> right-loc zip/node :tag))
27-
(recur (zip/replace right-loc replacement) true)
28-
29-
;; no action needs to be taken, move to the next node
30-
:else
31-
(recur right-loc false))
32-
(recur loc true))))))
33-
3412
(defn update-online-resource-url
3513
"Update the the URL for elements within OnlineResources in echo10
3614
granule metadata and return metadata."
@@ -57,5 +35,34 @@
5735

5836
online-resources (echo10-utils/update-online-resources parsed :url :url url-map)]
5937
(-> parsed
60-
(replace-in-tree :OnlineResources online-resources)
38+
(echo10-utils/replace-in-tree :OnlineResources online-resources)
39+
xml/indent-str)))
40+
41+
(defn update-online-access-url
42+
"Update the the URL for elements within OnlineAccesses in echo10
43+
granule metadata and return metadata."
44+
[concept links]
45+
(let [parsed (xml/parse-str (:metadata concept))
46+
url-map (apply merge (map #(hash-map (:from %) (:to %)) links))
47+
48+
existing-urls (map #(cx/string-at-path % [:URL])
49+
(cx/elements-at-path parsed [:OnlineAccessURLs :OnlineAccessURL]))
50+
_ (when-not (set/superset? (set existing-urls) (set (keys url-map)))
51+
(errors/throw-service-errors
52+
:invalid-data
53+
[(str "Update failed - please only specify URLs contained in the"
54+
" existing granule OnlineAccessURLs ["
55+
(string/join ", " (set/difference (set (keys url-map)) (set existing-urls)))
56+
"] were not found")]))
57+
58+
_ (when-not (= (count (set (keys url-map))) (count links))
59+
(errors/throw-service-errors
60+
:invalid-data
61+
[(str "Update failed - duplicate URLs provided for granule update ["
62+
(string/join ", " (for [[v freq] (frequencies (map :URL links)) :when (> freq 1)] v))
63+
"]")]))
64+
65+
online-access-urls (echo10-utils/update-online-access-urls parsed :url :url url-map)]
66+
(-> parsed
67+
(echo10-utils/replace-in-tree :OnlineAccessURLs online-access-urls)
6168
xml/indent-str)))

‎ingest-app/src/cmr/ingest/services/granule_bulk_update/utils/echo10.clj‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"Contains functions for updating ECHO10 granule xml metadata."
33
(:require
44
[clojure.data.xml :as xml]
5+
[clojure.zip :as zip]
56
[cmr.common.xml :as cx]))
67

78
(defn xml-elem->online-resource
@@ -43,3 +44,60 @@
4344
[:OnlineResources :OnlineResource])]
4445
(when (seq online-resources)
4546
(update-resources online-resources locator-field update-field value-map))))
47+
48+
(defn- xml-elem->online-access-url
49+
"Parses and returns XML element for OnlineAccessURL"
50+
[elem]
51+
(let [url (cx/string-at-path elem [:URL])
52+
description (cx/string-at-path elem [:URLDescription])
53+
mime-type (cx/string-at-path elem [:MimeType])]
54+
{:url url
55+
:url-description description
56+
:mime-type mime-type}))
57+
58+
(defn- update-accesses
59+
"Constructs the new OnlineAccessURLs node in zipper representation"
60+
[online-access-urls locator-field value-field value-map]
61+
(let [edn-access-urls (map xml-elem->online-access-url online-access-urls)
62+
access-urls (map #(merge %
63+
(when-let [replacement (get value-map (get % locator-field))]
64+
(hash-map value-field replacement)))
65+
edn-access-urls)]
66+
(xml/element
67+
:OnlineAccessURLs {}
68+
(for [r access-urls]
69+
(let [{:keys [url url-description mime-type]} r]
70+
(xml/element :OnlineAccessURL {}
71+
(xml/element :URL {} url)
72+
(when url-description (xml/element :URLDescription {} url-description))
73+
(when mime-type (xml/element :MimeType {} mime-type))))))))
74+
75+
(defn update-online-access-urls
76+
"Return an OnlineAccessURLs node where UPDATE-FIELD is updated where the
77+
LOCATOR-FIELD has a matching key in the VALUE-MAP."
78+
[xml-tree locator-field value-field value-map]
79+
(let [online-access-urls (cx/elements-at-path
80+
xml-tree
81+
[:OnlineAccessURLs :OnlineAccessURL])]
82+
(when (seq online-access-urls)
83+
(update-accesses online-access-urls locator-field value-field value-map))))
84+
85+
(defn replace-in-tree
86+
"Take a parsed granule xml, replace the given node with the provided replacement
87+
Returns the zipper representation of the updated xml."
88+
[tree element-tag replacement]
89+
(let [zipper (zip/xml-zip tree)
90+
start-loc (zip/down zipper)]
91+
(loop [loc start-loc done false]
92+
(if done
93+
(zip/root loc)
94+
(if-let [right-loc (zip/right loc)]
95+
(cond
96+
;; at an OnlineResources element, replace the node with updated value
97+
(= element-tag (-> right-loc zip/node :tag))
98+
(recur (zip/replace right-loc replacement) true)
99+
100+
;; no action needs to be taken, move to the next node
101+
:else
102+
(recur right-loc false))
103+
(recur loc true))))))

0 commit comments

Comments
 (0)