Error while downloading

openEO Platform
,
1

Hi,
I get an error when I’m trying to download a vegetation index from S2.

My bounding box is rather small:

bbox = {“west”: 5.05, “south”: 51.20, “east”: 5.15, “north”: 51.24,
“crs”: “EPSG:4326”}

Time frame is large:
dates = (“2018-01-01”, “2022-01-01”)

But I reduce it by:

red = cube.band(“B04”).reduce_dimension(dimension=“t”, reducer=“median”)
nir = cube.band(“B08”).reduce_dimension(dimension=“t”, reducer=“median”)

Then I calculate my index

sr = red / nir

To visualize the results, I read in the docs that I need to export into bytes formats
sr_bytes = sr.download(format=“GTIFF”)

But I get the following error:

ConnectionResetError Traceback (most recent call last)
/opt/conda/lib/python3.9/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
698 # Make the request on the httplib connection object.
→ 699 httplib_response = self._make_request(
700 conn,

/opt/conda/lib/python3.9/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
444 # Otherwise it looks like a bug in the code.
→ 445 six.raise_from(e, None)
446 except (SocketTimeout, BaseSSLError, SocketError) as e:

/opt/conda/lib/python3.9/site-packages/urllib3/packages/six.py in raise_from(value, from_value)

/opt/conda/lib/python3.9/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
439 try:
→ 440 httplib_response = conn.getresponse()
441 except BaseException as e:

/opt/conda/lib/python3.9/http/client.py in getresponse(self)
1348 try:
→ 1349 response.begin()
1350 except ConnectionError:

/opt/conda/lib/python3.9/http/client.py in begin(self)
315 while True:
→ 316 version, status, reason = self._read_status()
317 if status != CONTINUE:

/opt/conda/lib/python3.9/http/client.py in _read_status(self)
276 def _read_status(self):
→ 277 line = str(self.fp.readline(_MAXLINE + 1), “iso-8859-1”)
278 if len(line) > _MAXLINE:

/opt/conda/lib/python3.9/socket.py in readinto(self, b)
703 try:
→ 704 return self._sock.recv_into(b)
705 except timeout:

/opt/conda/lib/python3.9/ssl.py in recv_into(self, buffer, nbytes, flags)
1240 self.class)
→ 1241 return self.read(nbytes, buffer)
1242 else:

/opt/conda/lib/python3.9/ssl.py in read(self, len, buffer)
1098 if buffer is not None:
→ 1099 return self._sslobj.read(len, buffer)
1100 else:

ConnectionResetError: [Errno 104] Connection reset by peer

During handling of the above exception, another exception occurred:

ProtocolError Traceback (most recent call last)
/opt/conda/lib/python3.9/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
438 if not chunked:
→ 439 resp = conn.urlopen(
440 method=request.method,

/opt/conda/lib/python3.9/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
754
→ 755 retries = retries.increment(
756 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]

/opt/conda/lib/python3.9/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
531 if read is False or not self._is_method_retryable(method):
→ 532 raise six.reraise(type(error), error, _stacktrace)
533 elif read is not None:

/opt/conda/lib/python3.9/site-packages/urllib3/packages/six.py in reraise(tp, value, tb)
768 if value.traceback is not tb:
→ 769 raise value.with_traceback(tb)
770 raise value

/opt/conda/lib/python3.9/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
698 # Make the request on the httplib connection object.
→ 699 httplib_response = self._make_request(
700 conn,

/opt/conda/lib/python3.9/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
444 # Otherwise it looks like a bug in the code.
→ 445 six.raise_from(e, None)
446 except (SocketTimeout, BaseSSLError, SocketError) as e:

/opt/conda/lib/python3.9/site-packages/urllib3/packages/six.py in raise_from(value, from_value)

/opt/conda/lib/python3.9/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
439 try:
→ 440 httplib_response = conn.getresponse()
441 except BaseException as e:

/opt/conda/lib/python3.9/http/client.py in getresponse(self)
1348 try:
→ 1349 response.begin()
1350 except ConnectionError:

/opt/conda/lib/python3.9/http/client.py in begin(self)
315 while True:
→ 316 version, status, reason = self._read_status()
317 if status != CONTINUE:

/opt/conda/lib/python3.9/http/client.py in _read_status(self)
276 def _read_status(self):
→ 277 line = str(self.fp.readline(_MAXLINE + 1), “iso-8859-1”)
278 if len(line) > _MAXLINE:

/opt/conda/lib/python3.9/socket.py in readinto(self, b)
703 try:
→ 704 return self._sock.recv_into(b)
705 except timeout:

/opt/conda/lib/python3.9/ssl.py in recv_into(self, buffer, nbytes, flags)
1240 self.class)
→ 1241 return self.read(nbytes, buffer)
1242 else:

/opt/conda/lib/python3.9/ssl.py in read(self, len, buffer)
1098 if buffer is not None:
→ 1099 return self._sslobj.read(len, buffer)
1100 else:

ProtocolError: (‘Connection aborted.’, ConnectionResetError(104, ‘Connection reset by peer’))

During handling of the above exception, another exception occurred:

ConnectionError Traceback (most recent call last)
/tmp/ipykernel_1564/1664275815.py in
1 # To visualize the results, we need to export into bytes formats
----> 2 sr_bytes = sr.download(format=“GTIFF”)

/opt/conda/lib/python3.9/site-packages/openeo/rest/datacube.py in download(self, outputfile, format, options)
1612 # TODO: only add save_result node when there is none yet?
1613 cube = self.save_result(format=format, options=options)
→ 1614 return self._connection.download(cube.flat_graph(), outputfile)
1615
1616 def validate(self) → List[dict]:

/opt/conda/lib/python3.9/site-packages/openeo/rest/connection.py in download(self, graph, outputfile, timeout)
1108 “”"
1109 request = self._build_request_with_process_graph(process_graph=graph)
→ 1110 response = self.post(path=“/result”, json=request, expected_status=200, stream=True, timeout=timeout)
1111
1112 if outputfile is not None:

/opt/conda/lib/python3.9/site-packages/openeo/rest/connection.py in post(self, path, json, **kwargs)
171 :return: response: Response
172 “”"
→ 173 return self.request(“post”, path=path, json=json, allow_redirects=False, **kwargs)
174
175 def delete(self, path, **kwargs) → Response:

/opt/conda/lib/python3.9/site-packages/openeo/rest/connection.py in request(self, method, path, headers, auth, check_error, expected_status, **kwargs)
600 try:
601 # Initial request attempt
→ 602 return _request()
603 except OpenEoApiError as api_exc:
604 if api_exc.http_status_code == 403 and api_exc.code == “TokenInvalid”:

/opt/conda/lib/python3.9/site-packages/openeo/rest/connection.py in _request()
593 # Do request, but with retry when access token has expired and refresh token is available.
594 def _request():
→ 595 return super(Connection, self).request(
596 method=method, path=path, headers=headers, auth=auth,
597 check_error=check_error, expected_status=expected_status, **kwargs,

/opt/conda/lib/python3.9/site-packages/openeo/rest/connection.py in request(self, method, path, headers, auth, check_error, expected_status, **kwargs)
100 )
101 with ContextTimer() as timer:
→ 102 resp = self.session.request(
103 method=method,
104 url=url,

/opt/conda/lib/python3.9/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
540 }
541 send_kwargs.update(settings)
→ 542 resp = self.send(prep, **send_kwargs)
543
544 return resp

/opt/conda/lib/python3.9/site-packages/requests/sessions.py in send(self, request, **kwargs)
653
654 # Send the request
→ 655 r = adapter.send(request, **kwargs)
656
657 # Total elapsed time of the request (approximately)

/opt/conda/lib/python3.9/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
496
497 except (ProtocolError, socket.error) as err:
→ 498 raise ConnectionError(err, request=request)
499
500 except MaxRetryError as e:

ConnectionError: (‘Connection aborted.’, ConnectionResetError(104, ‘Connection reset by peer’))

Thanks in advance

2

Hi @dileomargherita ,
this looks like a network issue, but normally all backends should be online.
Can you simply try again?

If it persists, it would also be good to have the part of your code that sets up the connection, and the data collection that you try to use.

best regards,
Jeroen

3

I have tried multiple times over some days, but the response is the same.

Here’s the offending code:

import openeo

con = openeo.connect(“https://openeocloud.vito.be/openeo/1.0.0”).authenticate_oidc()

bbox = {“west”: 5.05, “south”: 51.20, “east”: 5.15, “north”: 51.24,
“crs”: “EPSG:4326”}
bands = [“B03”, “B04”, “B05”, “B08”, “B11”]
dates = (“2018-01-01”, “2022-01-01”)

cube = con.load_collection(“SENTINEL2_L2A_SENTINELHUB”,
spatial_extent=bbox,
temporal_extent=dates,
bands=bands)

Reduce dimension along time

green = cube.band(“B03”).reduce_dimension(dimension=“t”, reducer=“median”)
red = cube.band(“B04”).reduce_dimension(dimension=“t”, reducer=“median”)
redEdge = cube.band(“B05”).reduce_dimension(dimension=“t”, reducer=“median”)
nir = cube.band(“B08”).reduce_dimension(dimension=“t”, reducer=“median”)
swir1 = cube.band(“B11”).reduce_dimension(dimension=“t”, reducer=“median”)

Simple Ratio (SR)

sr = red / nir

sr_bytes = sr.download(format=“GTIFF”)

This latter yields the above mentioned error.

Thanks

4

Thanks, I can now reproduce it.
It seems that the request takes quite some time, so I think the error is actually our regular 15 minute timeout which gets translated badly.
Can you try running the same with ‘execute_batch’ instead of ‘download’?

The other option would be to try and get the job to run faster. You can for instance use ‘SENTINEL2_L2A’ as collection name, which falls back to a faster collection if possible, and you could limit the bands to the ones that you actually use, because now you probably trigger more data loading then needed.

best regards,
Jeroen

5

Thank you Jeroen.
I download all those bands because I need to calculate multiple vegetation indexes, not only the SR (simple ratio).

A couple of questions:

  1. the reduce_dimension step should really reduce also the size of the band that is actually used in the computation, so how come is requiring much resources?

  2. What are the differences between SENTINEL2_L2A_SENTINELHUB and SENTINEL2_L2A? from the catalogue I don’t have elements to pick the one or the other

Execute_batch works, however I am trying to visualize the result after I download it with this function (reused from somewhere in the documentation):

from rasterio.io import MemoryFile
def plot_result(data):
with MemoryFile(data) as memfile:
with memfile.open() as dataset:
data_array = dataset.read()
from rasterio.plot import show,show_hist
import matplotlib.pyplot as plt
fig, (l, r) = plt.subplots(1, 2,figsize=(12, 5))
show(data_array,cmap=‘YlGn’,ax=l)
show_hist(data_array,ax=r)

plot_result(sr_bytes)

But I get the following error:

TypeError Traceback (most recent call last)
Cell In[19], line 1
----> 1 plot_result(sr_bytes)

Cell In[17], line 2, in plot_result(data)
1 def plot_result(data):
----> 2 with MemoryFile(data) as memfile:
3 with memfile.open() as dataset:
4 data_array = dataset.read()

File /opt/conda/lib/python3.9/site-packages/rasterio/io.py:104, in MemoryFile.init(self, file_or_bytes, dirname, filename, ext)
88 def init(self, file_or_bytes=None, dirname=None, filename=None, ext=‘.tif’):
89 “”“Create a new file in memory
90
91 Parameters
(…)
102 MemoryFile
103 “””
→ 104 super().init(
105 file_or_bytes=file_or_bytes, dirname=dirname, filename=filename, ext=ext
106 )

File rasterio/_io.pyx:868, in rasterio._io.MemoryFileBase.init()

TypeError: Constructor argument must be a file opened in binary mode or bytes.

Thanks

6

Hi,

thanks for confirming that batch jobs work. The example code doesn’t work because it reads the geotiff from bytes in memory, while the batch job will usually be downloaded to disk and read from there.

So if you do cube.execute_batch('myfile.tif')
The result will be downloaded to myfile.tif, which you can for instance open with rasterio, as shown in this example:
https://rasterio.readthedocs.io/en/latest/topics/plotting.html

7

And to answer these questions:

The resource are mainly required to load the original timeseries, before reducing it.
The difference between the two collections is indeed not that clearly documented, but the one with ‘_SENTINELHUB’ suffix forces the use of Sentinelhub. The other one is a harmonized collection, which is described here:

The idea is in fact that in the future, we only need the harmonized version and can drop the backend specific collections entirely.