Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
(post 4.2.0 release)
### Added
- Support for new OGC endpoints
### Changed
### Deprecated
### Removed
- Removed functions calling old WPST endpoints
### Fixed
### Security

## [4.2.0]
### Added
### Changed
- listJobs no longer takes username as an argument, you can only list jobs for the current `MAAP_PGT` token user
- submitJob gets the username from the `MAAP_PGT` token and not username being submitted as an argument
- submit_job gets the username from the `MAAP_PGT` token and not username being submitted as an argument
### Deprecated
### Removed
### Fixed
Expand Down
10 changes: 6 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ $ python
>>> from maap.maap import MAAP
>>> maap = MAAP()

>>> granules = maap.searchGranule(sitename='lope', instrument='uavsar')
>>> granules = maap.search_granule(sitename='lope', instrument='uavsar')
>>> for res in granules:
print(res.getDownloadUrl())
res.download()
Expand Down Expand Up @@ -64,17 +64,19 @@ where:
With named attribute parameters, this query:

```python
lidarGranule = maap.searchGranule(instrument='lvis', attribute='string,Site Name,lope')
lidarGranule = maap.search_granule(instrument='lvis', attribute='string,Site Name,lope')
```

Simplifies to:

```python
lidarGranule = maap.searchGranule(instrument='lvis', site_name='lope')
lidarGranule = maap.search_granule(instrument='lvis', site_name='lope')
```

## Test

```bash
python setup.py test
poetry install
poetry run pytest --cov=maap
poetry run pytest test/specific_test.py
```
2 changes: 1 addition & 1 deletion docs/api/dps.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ from maap.maap import MAAP
maap = MAAP()

# Submit a job
job = maap.submitJob(
job = maap.submit_job(
identifier='my_analysis',
algo_id='my_algorithm',
version='main',
Expand Down
6 changes: 3 additions & 3 deletions docs/api/maap.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,13 @@ from maap.maap import MAAP
maap = MAAP()

# Search granules
granules = maap.searchGranule(short_name='GEDI02_A', limit=10)
granules = maap.search_granule(short_name='GEDI02_A', limit=10)

# Search collections
collections = maap.searchCollection(provider='MAAP')
collections = maap.search_collection(provider='MAAP')

# Submit a job
job = maap.submitJob(
job = maap.submit_job(
identifier='analysis',
algo_id='my_algo',
version='main',
Expand Down
2 changes: 1 addition & 1 deletion docs/api/result.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ from maap.maap import MAAP
maap = MAAP()

# Search granules
granules = maap.searchGranule(short_name='GEDI02_A', limit=5)
granules = maap.search_granule(short_name='GEDI02_A', limit=5)

for granule in granules:
# Get URLs
Expand Down
4 changes: 2 additions & 2 deletions docs/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ from maap.maap import MAAP
maap = MAAP()

# Search for granules
granules = maap.searchGranule(
granules = maap.search_granule(
short_name='GEDI02_A',
bounding_box='-122.5,37.5,-121.5,38.5',
limit=10
Expand All @@ -42,7 +42,7 @@ for granule in granules:
print(f"Downloaded: {local_path}")

# Submit a job
job = maap.submitJob(
job = maap.submit_job(
identifier='my_analysis',
algo_id='my_algorithm',
version='main',
Expand Down
4 changes: 2 additions & 2 deletions examples/BrowseExample.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@
}
],
"source": [
"granule = maap.searchGranule(granule_ur='uavsar_AfriSAR_v1_SLC-lopenp_14043_16015_001_160308_L090.vrt')[0]\n",
"granule = maap.search_granule(granule_ur='uavsar_AfriSAR_v1_SLC-lopenp_14043_16015_001_160308_L090.vrt')[0]\n",
"maap.show(granule)"
]
},
Expand Down Expand Up @@ -747,7 +747,7 @@
}
],
"source": [
"granule = maap.searchGranule(granule_ur='ILVIS2_GA2016_0220_R1611_038024')[0]\n",
"granule = maap.search_granule(granule_ur='ILVIS2_GA2016_0220_R1611_038024')[0]\n",
"maap.show(granule)"
]
}
Expand Down
2 changes: 1 addition & 1 deletion examples/Search Collection - Basics-checkpoint.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
"metadata": {},
"outputs": [],
"source": [
"results = maap.searchCollection(keyword='precipitation')"
"results = maap.search_collection(keyword='precipitation')"
]
},
{
Expand Down
4 changes: 2 additions & 2 deletions examples/Search Granule-checkpoint.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
}
],
"source": [
"results = maap.searchCollection(keyword='land',data_center='modaps')\n",
"results = maap.search_collection(keyword='land',data_center='modaps')\n",
"print(len(results))\n",
"\n",
"for res in results:\n",
Expand Down Expand Up @@ -159,7 +159,7 @@
}
],
"source": [
"results = maap.searchGranule(limit=10,short_name=\"MOD11A1\")\n",
"results = maap.search_granule(limit=10,short_name=\"MOD11A1\")\n",
"\n",
"print(len(results))\n",
"for res in results:\n",
Expand Down
2 changes: 1 addition & 1 deletion maap/AWS.py
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ def workspace_bucket_credentials(self):
See Also
--------
:meth:`requester_pays_credentials` : For accessing external data
:meth:`maap.maap.MAAP.uploadFiles` : Upload files to shared storage
:meth:`maap.maap.MAAP.upload_files` : Upload files to shared storage
"""
headers = self._api_header
headers["Accept"] = "application/json"
Expand Down
2 changes: 1 addition & 1 deletion maap/Profile.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ def account_info(self, proxy_ticket=None):

Notes
-----
This method is used internally by :meth:`~maap.maap.MAAP.submitJob`
This method is used internally by :meth:`~maap.maap.MAAP.submit_job`
to automatically include the username with job submissions.

See Also
Expand Down
10 changes: 5 additions & 5 deletions maap/Result.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from maap.maap import MAAP

maap = MAAP()
granules = maap.searchGranule(short_name='GEDI02_A', limit=5)
granules = maap.search_granule(short_name='GEDI02_A', limit=5)

for granule in granules:
# Get download URLs
Expand Down Expand Up @@ -439,7 +439,7 @@ class Collection(Result):
--------
Search for collections::

>>> collections = maap.searchCollection(short_name='GEDI02_A')
>>> collections = maap.search_collection(short_name='GEDI02_A')
>>> for c in collections:
... print(c['Collection']['ShortName'])
... print(c['Collection']['Description'])
Expand All @@ -458,7 +458,7 @@ class Collection(Result):
See Also
--------
:class:`Granule` : Individual data file results
:meth:`maap.maap.MAAP.searchCollection` : Search for collections
:meth:`maap.maap.MAAP.search_collection` : Search for collections
"""

def __init__(self, metaResult, maap_host):
Expand Down Expand Up @@ -512,7 +512,7 @@ class Granule(Result):
--------
Search and access granule metadata::

>>> granules = maap.searchGranule(short_name='GEDI02_A', limit=5)
>>> granules = maap.search_granule(short_name='GEDI02_A', limit=5)
>>> granule = granules[0]
>>> print(granule['Granule']['GranuleUR'])

Expand Down Expand Up @@ -540,7 +540,7 @@ class Granule(Result):
See Also
--------
:class:`Collection` : Dataset metadata results
:meth:`maap.maap.MAAP.searchGranule` : Search for granules
:meth:`maap.maap.MAAP.search_granule` : Search for granules
"""

def __init__(
Expand Down
4 changes: 2 additions & 2 deletions maap/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
maap = MAAP()

# Search for granules
granules = maap.searchGranule(
granules = maap.search_granule(
short_name='GEDI02_A',
limit=10
)
Expand All @@ -34,7 +34,7 @@
local_path = granule.getData(destpath='/tmp')

# Submit a job
job = maap.submitJob(
job = maap.submit_job(
identifier='my_job',
algo_id='my_algorithm',
version='main',
Expand Down
3 changes: 3 additions & 0 deletions maap/config_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,9 @@ def __init__(self, maap_host):
self.algorithm_build = self._get_api_endpoint("algorithm_build")
self.mas_algo = self._get_api_endpoint("mas_algo")
self.dps_job = self._get_api_endpoint("dps_job")
self.processes_ogc = self._get_api_endpoint("processes_ogc")
self.deployment_jobs_ogc = self._get_api_endpoint("deployment_jobs_ogc")
self.jobs_ogc = self._get_api_endpoint("jobs_ogc")
self.member_dps_token = self._get_api_endpoint("member_dps_token")
self.requester_pays = self._get_api_endpoint("requester_pays")
self.edc_credentials = self._get_api_endpoint("edc_credentials")
Expand Down
34 changes: 25 additions & 9 deletions maap/dps/dps_job.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
maap = MAAP()

# Submit a job
job = maap.submitJob(
job = maap.submit_job(
identifier='my_analysis',
algo_id='my_algorithm',
version='main',
Expand All @@ -38,13 +38,14 @@

See Also
--------
:meth:`maap.maap.MAAP.submitJob` : Submit a new job
:meth:`maap.maap.MAAP.getJob` : Retrieve an existing job
:meth:`maap.maap.MAAP.submit_job` : Submit a new job
:meth:`maap.maap.MAAP.get_job` : Retrieve an existing job
"""

import json
import logging
import os
import time
import xml.etree.ElementTree as ET
import backoff
from urllib.parse import urljoin
Expand Down Expand Up @@ -131,12 +132,12 @@ class DPSJob:
--------
Get job status::

>>> job = maap.getJob('f3780917-92c0-4440-8a84-9b28c2e64fa8')
>>> job = maap.get_job('f3780917-92c0-4440-8a84-9b28c2e64fa8')
>>> print(f"Status: {job.status}")

Wait for completion::

>>> job = maap.submitJob(...)
>>> job = maap.submit_job(...)
>>> job.wait_for_completion()
>>> print(f"Final status: {job.status}")

Expand All @@ -159,7 +160,7 @@ class DPSJob:

See Also
--------
:meth:`maap.maap.MAAP.submitJob` : Submit new jobs
:meth:`maap.maap.MAAP.submit_job` : Submit new jobs
:meth:`maap.maap.MAAP.listJobs` : List all jobs
"""

Expand Down Expand Up @@ -227,13 +228,20 @@ def retrieve_status(self):
return self.status

@backoff.on_exception(backoff.expo, Exception, max_value=64, max_time=172800)
def wait_for_completion(self):
def wait_for_completion(self, initial_delay_seconds=5):
"""
Wait for the job to complete.

Blocks execution until the job finishes (succeeds, fails, or is
cancelled). Uses exponential backoff to poll for status updates.

Parameters
----------
initial_delay_seconds : float, optional
Initial delay in seconds before first status check (default: 5).
This accounts for the time required for newly submitted jobs to
become visible in the database. Set to 0 to disable initial delay.

Returns
-------
DPSJob
Expand All @@ -243,7 +251,7 @@ def wait_for_completion(self):
--------
::

>>> job = maap.submitJob(...)
>>> job = maap.submit_job(...)
>>> job.wait_for_completion()
>>> if job.status == 'Succeeded':
... print("Job completed successfully!")
Expand All @@ -254,14 +262,22 @@ def wait_for_completion(self):
- Uses exponential backoff with max interval of 64 seconds
- Maximum wait time is 48 hours (172800 seconds)
- The job object is updated with final status upon completion
- Initial delay accounts for database visibility delay (~5 seconds)

See Also
--------
:meth:`retrieve_status` : Check status without blocking
:meth:`cancel_job` : Cancel a running job
"""
# Wait before first check to allow newly submitted jobs to appear in database
if initial_delay_seconds > 0:
logger.debug(f'Waiting {initial_delay_seconds} seconds before first status check')
time.sleep(initial_delay_seconds)

self.retrieve_status()
if self.status.lower() in ["accepted", "running"]:
# Known terminal states and states that should trigger retries
terminal_states = ["succeeded", "failed", "dismissed", "deduped", "offline"]
if self.status.lower() not in terminal_states:
logger.debug('Current Status is {}. Backing off.'.format(self.status))
raise RuntimeError
return self
Expand Down
Loading
Loading