Skip to content

Commit 273ebe6

Browse files
rhubertrhubert
committed
wip
1 parent b05cd0d commit 273ebe6

4 files changed

Lines changed: 464 additions & 433 deletions

File tree

‎pym/bob/archive.py‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -449,8 +449,8 @@ def _downloadPackage(self, buildId, suffix, audit, content, caches, workspace):
449449
return (False, self._namedErrorString("not found"), WARNING)
450450
except (ArtifactDownloadError, HttpDownloadError) as e:
451451
return (False, self._namedErrorString(e.reason), WARNING)
452-
except ConnectionRefusedError:
453-
return (False, self._namedErrorString("connection failed"), WARNING)
452+
except (ConnectionRefusedError, urllib.error.URLError) as e:
453+
return (False, self._namedErrorString("connection failed" + str(e)), WARNING)
454454
except (socket.herror, socket.gaierror) as e:
455455
return (False, self._namedErrorString(str(e)), WARNING)
456456
except BuildError as e:
@@ -491,7 +491,7 @@ def _downloadLocalFile(self, key, suffix):
491491
return (None, self._namedErrorString("not found"), WARNING)
492492
except (ArtifactDownloadError, HttpDownloadError) as e:
493493
return (None, self._namedErrorString(e.reason), WARNING)
494-
except ConnectionRefusedError:
494+
except (ConnectionRefusedError, urllib.error.URLError):
495495
return (None, self._namedErrorString("connection failed"), WARNING)
496496
except (socket.herror, socket.gaierror) as e:
497497
return (None, self._namedErrorString(str(e)), WARNING)
@@ -878,6 +878,7 @@ def __exit__(self, exc_type, exc_value, traceback):
878878
class HttpArchive(BaseArchive):
879879
def __init__(self, spec):
880880
super().__init__(spec)
881+
print("init url: " + spec["url"])
881882
self.__url = urllib.parse.urlparse(spec["url"])
882883
self._webdav = WebDav(self.__url, spec.get("sslVerify", True))
883884
self._retries = spec.get("retries", 1)
@@ -895,17 +896,13 @@ def __retry(self, request):
895896
while True:
896897
try:
897898
return request()
898-
except (HTTPException, OSError) as e:
899-
self._webdav._resetConnection()
899+
except (HTTPException, OSError, urllib.error.URLError) as e:
900900
if retries == 0: raise e
901901
retries -= 1
902902

903903
def _canManage(self):
904904
return True
905905

906-
def _resetConnection(self):
907-
self._webdav._resetConnection()
908-
909906
def _makePath(self, buildId, suffix):
910907
packageResultId = buildIdToName(buildId)
911908
return "/".join([self.__url.path, packageResultId[0:2], packageResultId[2:4],
@@ -991,9 +988,6 @@ def __init__(self, archiver, response):
991988
def __enter__(self):
992989
return (None, self.response)
993990
def __exit__(self, exc_type, exc_value, traceback):
994-
# reset connection on abnormal termination
995-
if exc_type is not None:
996-
self.archiver._resetConnection()
997991
return False
998992

999993

@@ -1295,6 +1289,7 @@ async def downloadLocalFingerprint(self, step, key, executor=None):
12951289

12961290
def getSingleArchiver(recipes, archiveSpec):
12971291
archiveBackend = archiveSpec.get("backend", "none")
1292+
print(archiveSpec)
12981293
if archiveBackend == "file":
12991294
return LocalArchive(archiveSpec)
13001295
elif archiveBackend == "http":

‎pym/bob/webdav.py‎

Lines changed: 110 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
from .utils import sslNoVerifyContext
44
import base64
55
import http.client
6+
import sys
67
import os
7-
from urllib.parse import unquote, urlsplit
8+
import urllib.request
9+
from urllib.parse import unquote, urlsplit, urlunparse
810
from xml.etree.ElementTree import fromstring
911

1012
class HTTPException(Exception):
@@ -47,30 +49,24 @@ def __init__(self, url, sslVerify=True):
4749
self.__connection = None
4850
self.__sslVerify = sslVerify
4951

50-
def getPartialDownloader(self, path, length=512*1024):
51-
return self.PartialDownloader(self, path, length)
52+
if self.__url.username is not None:
5253

53-
def _getConnection(self):
54-
if self.__connection is not None:
55-
return self.__connection
54+
new_netloc = self.__url.hostname
55+
if self.__url.port is not None:
56+
new_netloc += ":" + str(self.__url.port)
57+
url = self.__url._replace(netloc=new_netloc)
58+
print(f"UNPARSED URL: {urlunparse(url)}")
59+
pmgr = urllib.request.HTTPPasswordMgr()
60+
pmgr.add_password(None, urlunparse(url), self.__url.username, self.__url.password)
5661

57-
url = self.__url
58-
if url.scheme == 'http':
59-
connection = http.client.HTTPConnection(url.hostname, url.port)
60-
elif url.scheme == 'https':
61-
ctx = None if self.__sslVerify else sslNoVerifyContext()
62-
connection = http.client.HTTPSConnection(url.hostname, url.port,
63-
context=ctx)
64-
else:
65-
raise BuildError("Unsupported URL scheme: '{}'".format(url.scheme))
62+
handler = urllib.request.HTTPBasicAuthHandler(pmgr)
63+
opener = urllib.request.build_opener(handler)
64+
urllib.request.install_opener(opener)
6665

67-
self.__connection = connection
68-
return connection
66+
self.__url = url
6967

70-
def _resetConnection(self):
71-
if self.__connection is not None:
72-
self.__connection.close()
73-
self.__connection = None
68+
def getPartialDownloader(self, path, length=512*1024):
69+
return self.PartialDownloader(self, path, length)
7470

7571
def _getHeaders(self):
7672
headers = {'User-Agent': 'BobBuildTool/{}'.format(BOB_VERSION)}
@@ -83,33 +79,38 @@ def _getHeaders(self):
8379
return headers
8480

8581
def exists(self, path):
86-
connection = self._getConnection()
87-
connection.request("HEAD", path, headers=self._getHeaders())
88-
response = connection.getresponse()
89-
response.read()
90-
if response.status == 200:
91-
return True
92-
elif response.status == 404:
93-
return False
94-
else:
95-
raise HttpUploadError("HEAD {} {}".format(response.status, response.reason))
82+
print(f"YYYYY>>>>>>>>>>>>>> {urlunparse(self.__url) + path}")
83+
req = urllib.request.Request (urlunparse(self.__url) + path,
84+
headers= self._getHeaders(), method="HEAD")
85+
ctx = None if self.__sslVerify else sslNoVerifyContext()
86+
try:
87+
with urllib.request.urlopen (req, context=ctx) as response:
88+
if response.status == 200:
89+
return True
90+
except urllib.error.HTTPError as e:
91+
e.close()
92+
if e.status == 404:
93+
return False
94+
else:
95+
raise HttpUploadError("HEAD {} {}".format(e.status, e.reason))
9696

9797
def download(self, path, offset=None, length=None):
98-
connection = self._getConnection()
9998
headers = self._getHeaders()
10099
if offset is not None and length is not None:
101100
headers.update({'Range': 'bytes={}-{}'.format(offset, offset + length - 1)})
102-
connection.request("GET", path, headers=headers)
103-
response = connection.getresponse()
104-
if response.status in [200, 206]:
105-
return response
106-
else:
107-
response.read()
108-
if response.status == 404:
101+
102+
req = urllib.request.Request (urlunparse(self.__url) + path,
103+
headers=headers, method="GET")
104+
ctx = None if self.__sslVerify else sslNoVerifyContext()
105+
try:
106+
return urllib.request.urlopen (req, context=ctx)
107+
except urllib.error.HTTPError as e:
108+
e.close()
109+
if e.status == 404:
109110
raise HttpNotFoundError()
110111
else:
111-
raise HttpDownloadError("{} {}".format(response.status,
112-
response.reason))
112+
raise HttpDownloadError("{} {}".format(e.status,
113+
e.reason))
113114

114115
def upload(self, path, buf, overwrite):
115116
# Determine file length ourselves and add a "Content-Length" header. This
@@ -121,55 +122,69 @@ def upload(self, path, buf, overwrite):
121122
headers.update({'Content-Length': length})
122123
if not overwrite:
123124
headers.update({'If-None-Match': '*'})
124-
connection = self._getConnection()
125-
connection.request("PUT", path, buf, headers=headers)
126-
response = connection.getresponse()
127-
response.read()
128-
if response.status == 412:
129-
# precondition failed -> lost race with other upload
130-
raise HttpAlreadyExistsError()
131-
elif response.status not in [200, 201, 204]:
132-
raise HttpUploadError("PUT {} {}".format(response.status, response.reason))
125+
126+
print(f"YYYYY>>>>>>>>>>>>>> {urlunparse(self.__url) + path}")
127+
req = urllib.request.Request (urlunparse(self.__url) + path,
128+
data=buf, headers=headers, method="PUT")
129+
ctx = None if self.__sslVerify else sslNoVerifyContext()
130+
try:
131+
urllib.request.urlopen (req, context=ctx)
132+
except urllib.error.HTTPError as e:
133+
e.close()
134+
if e.status == 412:
135+
# precondition failed -> lost race with other upload
136+
raise HttpAlreadyExistsError()
137+
elif e.status not in [200, 201, 204]:
138+
raise HttpUploadError("PUT {} {}".format(e.status, e.reason))
133139

134140
def _mkdir(self, path):
135141
# MKCOL resources must have a trailing slash because they are
136142
# directories. Otherwise Apache might send a HTTP 301. Nginx refuses to
137143
# create the directory with a 409 which looks odd.
138144
if not path.endswith("/"):
139145
path += "/"
140-
connection = self._getConnection()
141-
connection.request("MKCOL", path, headers=self._getHeaders())
142-
response = connection.getresponse()
143-
response.read()
144-
return response
146+
147+
req = urllib.request.Request (urlunparse(self.__url) + path,
148+
headers=self._getHeaders(), method="MKCOL")
149+
ctx = None if self.__sslVerify else sslNoVerifyContext()
150+
try:
151+
with urllib.request.urlopen (req, context=ctx) as resp:
152+
return (resp.status, None)
153+
except urllib.error.HTTPError as e:
154+
e.close()
155+
return (e.status, e.reason)
145156

146157
def mkdir(self, path, depth=1):
147158
if depth > 0:
148-
response = self._mkdir(path)
149-
if response.status == 409:
159+
status, reason = self._mkdir(path)
160+
if status == 409:
150161
(_path, _, _) = path.rpartition("/")
151162
self.mkdir(_path, depth - 1)
152-
response = self._mkdir(path)
163+
status,reason = self._mkdir(path)
153164
# We expect to create the directory (201) or it already existed (405).
154165
# If the server does not support MKCOL we'd expect a 405 too and hope
155166
# for the best...
156-
if response.status not in [201, 405]:
157-
raise HttpUploadError("MKCOL {} {}".format(response.status, response.reason))
167+
if status not in [201, 405]:
168+
raise HttpUploadError("MKCOL {} {}".format(status, reason))
158169

159170
def listdir(self, path):
160171
base_path = self.__url.path
161-
# create a full path ending with trailing / (should prevent http 301 - moved permanently)
162-
path = '/'.join([base_path, path.strip('/'), ''])
163172
if self.exists(path):
164173
headers = self._getHeaders()
165174
# Depth: 1 - applies to the resource and the immediate children (infinity usually prohibited by server)
166175
headers.update({'Depth': '1'})
167-
connection = self._getConnection()
168-
connection.request("PROPFIND", path, headers=headers)
169-
response = connection.getresponse()
170-
if response.status not in [207]:
171-
raise HttpDownloadError("PROPFIND {} {}".format(response.status, response.reason))
172-
content = response.read()
176+
req = urllib.request.Request (urlunparse(self.__url) + path,
177+
headers=headers, method="PROPFIND")
178+
ctx = None if self.__sslVerify else sslNoVerifyContext()
179+
content = None
180+
try:
181+
with urllib.request.urlopen (req, context=ctx) as response:
182+
if response.status not in [207]:
183+
raise HttpDownloadError("PROPFIND {} {}".format(response.status, response.reason))
184+
content = response.read()
185+
except urllib.error.HTTPError as e:
186+
e.close()
187+
raise HttpDownloadError("PROPFIND {} {}".format(e.status, e.reason))
173188
# get all dav responses from multistatusresponse
174189
tree = fromstring(content)
175190
dir_infos = []
@@ -192,12 +207,19 @@ def delete(self, filename):
192207
# create a full path
193208
filepath = '/'.join([base_path, filename.strip('/')])
194209
headers = self._getHeaders()
195-
connection = self._getConnection()
196-
connection.request("DELETE", filepath, headers=headers)
197-
response = connection.getresponse()
198-
response.read()
199-
if response.status not in [200, 204, 404]:
200-
raise HttpDownloadError("DELETE {} {}".format(response.status, response.reason))
210+
req = urllib.request.Request (urlunparse(self.__url) + filepath,
211+
headers=headers, method="DELETE")
212+
status = reason = None
213+
ctx = None if self.__sslVerify else sslNoVerifyContext()
214+
try:
215+
with urllib.request.urlopen (req, context=ctx) as response:
216+
status = response.status
217+
except urllib.error.HTTPError as e:
218+
e.close()
219+
status = e.status
220+
reason = e.reason
221+
if status not in [200, 204, 404]:
222+
raise HttpDownloadError("DELETE {} {}".format(status, reason))
201223

202224
def stat(self, file):
203225
base_path = self.__url.path
@@ -207,13 +229,20 @@ def stat(self, file):
207229
headers = self._getHeaders()
208230
# Depth: 0 - applies to the resource itself
209231
headers.update({'Depth': '0'})
210-
connection = self._getConnection()
211-
connection.request("PROPFIND", filepath, headers=headers)
212-
response = connection.getresponse()
213-
if response.status not in [207]:
214-
raise HttpDownloadError("PROPFIND {} {}".format(response.status, response.reason))
215-
# get response
216-
content = response.read()
232+
233+
req = urllib.request.Request (urlunparse(self.__url) + filepath,
234+
headers=headers, method="PROPFIND")
235+
content = None
236+
try:
237+
with urllib.request.urlopen (req) as response:
238+
if response.status not in [207]:
239+
raise HttpDownloadError("PROPFIND {} {}".format(response.status, response.reason))
240+
# get response
241+
content = response.read()
242+
except urllib.error.HTTPError as e:
243+
e.close()
244+
raise HttpDownloadError("PROPFIND {} {}".format(e.status, e.reason))
245+
217246
# parse tree from content
218247
tree = fromstring(content)
219248
# get response tag tree

0 commit comments

Comments
 (0)