Skip to content

Mb force imprint #298

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mbuchhorn merged 6 commits into from
Sep 15, 2026
Merged

Mb force imprint #298

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
60918eb
Add geometry retrieval from STAC assets: implement `get_modelID_asset…
mbuchhorn Sep 13, 2026
b6785a9
Add UDF for forcing and masking probabilities: implement `udf_force_i…
mbuchhorn Sep 13, 2026
d0198f6
Add `delete_collection_item` utility: enable deletion of individual c…
mbuchhorn Sep 15, 2026
4bc5346
Add notebook for deleting items from `modelSTAC` collection: implemen…
mbuchhorn Sep 15, 2026
c34954d
Add `query_modelID_output_bands`: implement STAC query for retrieving…
mbuchhorn Sep 15, 2026
5d2e8d9
Merge pull request #297 from ESA-WEED-project/dev
mbuchhorn Sep 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
331 changes: 331 additions & 0 deletions notebooks/test_delete_items_from_collection.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,331 @@
{
"cells": [
{
"metadata": {},
"cell_type": "markdown",
"source": "# tests to delete unwanted modelIDs from the modelSTAC",
"id": "7187881d346266b1"
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2026-09-15T08:25:06.246567Z",
"start_time": "2026-09-15T08:24:59.582135700Z"
}
},
"cell_type": "code",
"source": [
"import pystac_client\n",
"import pandas as pd\n",
"from eo_processing.utils.storage import WEED_storage"
],
"id": "187fe67715113a74",
"outputs": [],
"execution_count": 1
},
{
"cell_type": "code",
"id": "initial_id",
"metadata": {
"collapsed": true,
"ExecuteTime": {
"end_time": "2026-09-15T08:25:18.518515200Z",
"start_time": "2026-09-15T08:25:18.506034800Z"
}
},
"source": [
"username = 'VAULT_TOKEN' # or your terrascope user name to hand over credentials manually\n",
"# details to STAC\n",
"stac_url:str = 'https://catalogue.weed.apex.esa.int'\n",
"collection_id:str = 'model-STAC'"
],
"outputs": [],
"execution_count": 2
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2026-09-15T08:25:22.668653500Z",
"start_time": "2026-09-15T08:25:20.543609200Z"
}
},
"cell_type": "code",
"source": [
"# get all itemIDs from modelSTAC\n",
"client = pystac_client.Client.open(stac_url)\n",
"\n",
"search = client.search(\n",
" collections=[collection_id],\n",
" fields=[\"id\", \"properties.topology\", \"properties.training_year\", \"properties.model_version\", \"properties.name_spatial_region\", \"properties.name_spatial_zone\", \"properties.modelID\"],\n",
")\n",
"\n",
"# run the search\n",
"results = []\n",
"for item in search.items_as_dicts():\n",
" results.append([item['id'], item['properties']['modelID'], item['properties']['topology'], item['properties']['training_year'], item['properties']['model_version'], item['properties']['name_spatial_region'], item['properties']['name_spatial_zone']])\n",
"\n",
"df_result = pd.DataFrame(results, columns=['item_id', 'modelID', 'typology', 'training_year', 'model_version', 'spatial_region', 'spatial_zone'])"
],
"id": "15d488e9fe1470d0",
"outputs": [],
"execution_count": 3
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2026-09-15T08:25:25.181893800Z",
"start_time": "2026-09-15T08:25:25.131787900Z"
}
},
"cell_type": "code",
"source": [
"# get the overview\n",
"df_result.head()"
],
"id": "7e0052d61288cea",
"outputs": [
{
"data": {
"text/plain": [
" item_id modelID typology \\\n",
"0 IUCNGET_global_v321_2024_IND IUCNGET_global_v321_2024_IND IUCNGET \n",
"1 IUCNGET_global_v315_2024_AFR IUCNGET_global_v315_2024_AFR IUCNGET \n",
"2 IUCNGET_global_v314_2024_NEO IUCNGET_global_v314_2024_NEO IUCNGET \n",
"3 IUCNGET_global_v314_2024_IND IUCNGET_global_v314_2024_IND IUCNGET \n",
"4 IUCNGET_global_v314_2024_AFR IUCNGET_global_v314_2024_AFR IUCNGET \n",
"\n",
" training_year model_version spatial_region spatial_zone \n",
"0 2024 3.21 global IndoMalesian \n",
"1 2024 3.15 global African \n",
"2 2024 3.14 global Neotropical \n",
"3 2024 3.14 global IndoMalesian \n",
"4 2024 3.14 global African "
],
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>item_id</th>\n",
" <th>modelID</th>\n",
" <th>typology</th>\n",
" <th>training_year</th>\n",
" <th>model_version</th>\n",
" <th>spatial_region</th>\n",
" <th>spatial_zone</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>IUCNGET_global_v321_2024_IND</td>\n",
" <td>IUCNGET_global_v321_2024_IND</td>\n",
" <td>IUCNGET</td>\n",
" <td>2024</td>\n",
" <td>3.21</td>\n",
" <td>global</td>\n",
" <td>IndoMalesian</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>IUCNGET_global_v315_2024_AFR</td>\n",
" <td>IUCNGET_global_v315_2024_AFR</td>\n",
" <td>IUCNGET</td>\n",
" <td>2024</td>\n",
" <td>3.15</td>\n",
" <td>global</td>\n",
" <td>African</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>IUCNGET_global_v314_2024_NEO</td>\n",
" <td>IUCNGET_global_v314_2024_NEO</td>\n",
" <td>IUCNGET</td>\n",
" <td>2024</td>\n",
" <td>3.14</td>\n",
" <td>global</td>\n",
" <td>Neotropical</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>IUCNGET_global_v314_2024_IND</td>\n",
" <td>IUCNGET_global_v314_2024_IND</td>\n",
" <td>IUCNGET</td>\n",
" <td>2024</td>\n",
" <td>3.14</td>\n",
" <td>global</td>\n",
" <td>IndoMalesian</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>IUCNGET_global_v314_2024_AFR</td>\n",
" <td>IUCNGET_global_v314_2024_AFR</td>\n",
" <td>IUCNGET</td>\n",
" <td>2024</td>\n",
" <td>3.14</td>\n",
" <td>global</td>\n",
" <td>African</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"execution_count": 4
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2026-09-15T08:25:26.828837600Z",
"start_time": "2026-09-15T08:25:26.806822200Z"
}
},
"cell_type": "code",
"source": [
"# selecte items to delete\n",
"df_del = df_result[~(df_result['typology'] == 'IUCNGET')]\n",
"\n",
"ldelete = df_del[df_del['model_version'] < 3].item_id.tolist()"
],
"id": "73903f37bab568f",
"outputs": [],
"execution_count": 5
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2026-09-15T08:25:28.072842500Z",
"start_time": "2026-09-15T08:25:28.011634200Z"
}
},
"cell_type": "code",
"source": "ldelete",
"id": "fea9dbbc06a3d4b8",
"outputs": [
{
"data": {
"text/plain": [
"['EUNIS2021plus_panEU_v201_2024_OneZone',\n",
" 'EUNIS2021plus_EU_v1_2024_STE',\n",
" 'EUNIS2021plus_EU_v1_2024_PAN',\n",
" 'EUNIS2021plus_EU_v1_2024_MED',\n",
" 'EUNIS2021plus_EU_v1_2024_CON',\n",
" 'EUNIS2021plus_EU_v1_2024_BOR',\n",
" 'EUNIS2021plus_EU_v1_2024_ATL',\n",
" 'EUNIS2021plus_EU_v1_2024_ALP']"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"execution_count": 6
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2026-09-15T08:25:29.386166Z",
"start_time": "2026-09-15T08:25:29.308068100Z"
}
},
"cell_type": "code",
"source": [
"# init the storage object\n",
"st = WEED_storage(username=username, s3_bucket='model', project='WEED', stac_env='prod')"
],
"id": "4d5fe40f5ffc5142",
"outputs": [],
"execution_count": 7
},
{
"metadata": {
"ExecuteTime": {
"end_time": "2026-09-15T08:26:50.696816200Z",
"start_time": "2026-09-15T08:26:48.614625600Z"
}
},
"cell_type": "code",
"source": [
"# now we delete the items we do not want\n",
"for item in ldelete:\n",
" print(f'delete item: {item}')\n",
" st.delete_collection_item(collection_name=collection_id, item_id=item)"
],
"id": "66216cd955542da7",
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"delete item: EUNIS2021plus_panEU_v201_2024_OneZone\n",
"Item 'EUNIS2021plus_panEU_v201_2024_OneZone' deleted successfully from collection 'model-STAC'\n",
"delete item: EUNIS2021plus_EU_v1_2024_STE\n",
"Item 'EUNIS2021plus_EU_v1_2024_STE' deleted successfully from collection 'model-STAC'\n",
"delete item: EUNIS2021plus_EU_v1_2024_PAN\n",
"Item 'EUNIS2021plus_EU_v1_2024_PAN' deleted successfully from collection 'model-STAC'\n",
"delete item: EUNIS2021plus_EU_v1_2024_MED\n",
"Item 'EUNIS2021plus_EU_v1_2024_MED' deleted successfully from collection 'model-STAC'\n",
"delete item: EUNIS2021plus_EU_v1_2024_CON\n",
"Item 'EUNIS2021plus_EU_v1_2024_CON' deleted successfully from collection 'model-STAC'\n",
"delete item: EUNIS2021plus_EU_v1_2024_BOR\n",
"Item 'EUNIS2021plus_EU_v1_2024_BOR' deleted successfully from collection 'model-STAC'\n",
"delete item: EUNIS2021plus_EU_v1_2024_ATL\n",
"Item 'EUNIS2021plus_EU_v1_2024_ATL' deleted successfully from collection 'model-STAC'\n",
"delete item: EUNIS2021plus_EU_v1_2024_ALP\n",
"Item 'EUNIS2021plus_EU_v1_2024_ALP' deleted successfully from collection 'model-STAC'\n"
]
}
],
"execution_count": 8
},
{
"metadata": {},
"cell_type": "code",
"outputs": [],
"execution_count": null,
"source": "",
"id": "857e74f1803a5361"
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading
Loading