BR-CONAB: the portal's new download URLs, and no slivers under 100 m²

CHANGELOG.md

@@ -54,6 +54,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Converters now publish `metrics:area` by default (`area_calculate_missing = True`, #277): measured from the geometry for every row if the source has no area, and for the rows where it is empty or 0 otherwise. Set `area_calculate_missing = False` to opt out.
 - Renamed the HCAT mapping attributes of `AddHCATMixin`, as they work with any HCAT mapping, not only EuroCrops: the `ec_mapping*` attributes use the `hcat_mapping*` prefix now. The helpers `load_ec_mapping()` and `ec_url()` are now `load_hcat_mapping()` and `hcat_mapping_url()` in `fiboa_cli.datasets.commons.hcat`; the copies in `fiboa_cli.datasets.commons.ec` were removed.
 - BE-VLG: Extended editions to 2018-2026 and aligned determination dates with the selected campaign year.
+- BR-CONAB: Reads the 29 mappings from the portal's new download URLs; ids no longer carry the file's upload timestamp, and polygons under 100 m² (digitising slivers, a fifth of Minas Gerais coffee) are left out.
 - CZ: Extended year coverage, including GPZ_DP editions (2019-2022), and added 2026 nested-archive support.
 - DE-SH: Extended support to editions 2023, 2025 and 2026.
 - DK: Editions now cover 2008-2026. The 2008 and 2009 editions are published without the crop and HCAT extensions because the source has no crop columns.

fiboa_cli/datasets/br_conab.py

@@ -1,44 +1,47 @@
 import math
+import re
 from pathlib import Path
 
+import pandas as pd
 from vecorel_cli.conversion.admin import AdminConverterMixin
 
 from ..conversion.fiboa_converter import FiboaBaseConverter
 
+MIN_FIELD_AREA = 100  # m²
+
 
 class Converter(AdminConverterMixin, FiboaBaseConverter):
+    # The portal's file list: https://barramento.conab.gov.br/portal-informacao-api/api/v1/download
     _sources = [
-        "Algodao/GO/ALGODAO_GO_Safra_2019_2020.zip",
-        "Algodao/GO/ALGODAO-GO_Safra_2018_2019.zip",
-        "Algodao/GO/GO_ALGODAO_2021.zip",
-        "Algodao/GO/GO_ALGODAO_2223.zip",
-        "Algodao/MS/MS_ALGODAO_2021.zip",
-        "Algodao/MS/MS_ALGODAO_2122.zip",
-        "Arroz_Irrigado/GO/GO_ARROZ_IRRIG_2122.zip",
-        "Arroz_Irrigado/GO/GO_ARROZ_IRRIG_INUND_2324.zip",
-        "Arroz_Irrigado/GO/ARROZ-GO_Safra_2018_2019.zip",
-        "Arroz_Irrigado/MS/ARROZ-MS_Safra_2018_2019.zip",
-        "Arroz_Irrigado/PR/ARROZ-PR_Safra_2017_2018.zip",
-        "Arroz_Irrigado/RS/ARROZ-RS_Safra_2019_2020.zip",
-        "Arroz_Irrigado/SC/ARROZ-SC_Safra_2018_2019.zip",
-        "Arroz_Irrigado/TO/ARROZ-TO_Safra_2017_2018.zip",
-        "Arroz_Irrigado/GO/GO_ARROZ_IRRIG_INUND_2324.zip",
-        "Arroz_Irrigado/TO/TO_ARROZ_IRRIG_2324.zip",
-        "Cana/GO/CANA-GO_Safra_2011_2012.zip",
-        "Cafe/BA/CAFE-BA_Safra_2019.zip",
-        "Cafe/DF/CAFE-DF_Safra_2018.zip",
-        "Cafe/GO/CAFE-GO_Safra_2018.zip",
-        "Cafe/GO/CAFE-GO_Safra_2019.zip",
-        "Cafe/PR/CAFE-PR_Safra_2017.zip",
-        "Cafe/MG/CAFE-MG_Safra_2017.zip",
-        "Cafe/DF/DF_CAFE_24.zip",
-        "Cafe/DF/DF_CAFE_24.zip",
-        "Cafe/GO/GO_CAFE_21.zip",
-        "Cafe/RJ/RJ_CAFE_21.zip",
-        "Culturas_de_Verao_1_Safra/DF/CV-DF_Safra_2013_2014.zip",
-        "Culturas_de_Verao_1_Safra/DF/CV-DF_Safra_2014_2015.zip",
-        "Culturas_de_Verao_1_Safra/DF/CV-DF_Safra_2017_2018.zip",
-        "Culturas_de_Verao_1_Safra/TO/CV-TO_Safra_2019_2020.zip",
+        "1784839922_algodao-go-safra-2019-2020.zip",
+        "1784839955_algodao-go-safra-2018-2019.zip",
+        "1784839890_go-algodao-2021.zip",
+        "1784558283_go-algodao-2223.zip",
+        "1784840252_ms-algodao-2021.zip",
+        "1784840209_ms-algodao-2122.zip",
+        "1784840479_go-arroz-irrig-2122.zip",
+        "1784840507_go-arroz-irrig-inund-23241.zip",
+        "1784840800_arroz-go-safra-2018-2019.zip",
+        "1784840902_arroz-ms-safra-2018-2019.zip",
+        "1784841005_arroz-pr-safra-2017-2018.zip",
+        "1784841189_arroz-rs-safra-2019-2020.zip",
+        "1784841407_arroz-sc-safra-2018-2019.zip",
+        "1784841526_arroz-to-safra-2017-2018.zip",
+        "1784841496_to-arroz-irrig-2324.zip",
+        "1785258893_cana-go-11-12.zip",
+        "1789672175_cafe-ba-19.zip",
+        "1785251263_cafe-df-18.zip",
+        "1785251314_cafe-go-18.zip",
+        "1785251334_cafe-go-19.zip",
+        "1785251366_cafe-pr-17.zip",
+        "1787760927_cafe-mg-safra-2017.zip",
+        "1785251294_cafe-df-24.zip",
+        "1785251350_cafe-go-21.zip",
+        "1785251708_cafe-rj-21.zip",
+        "1785335351_cv-df-safra-2013-2014.zip",
+        "1785335351_cv-df-safra-2014-2015.zip",
+        "1785335352_cv-df-safra-2017-2018.zip",
+        "1785335410_cv-to-safra-2019-2020.zip",
     ]
     sources = {
         "https://portaldeinformacoes.conab.gov.br/downloads/mapas/" + k: ["*.shp"] for k in _sources
@@ -49,7 +52,7 @@ class Converter(AdminConverterMixin, FiboaBaseConverter):
     description = """
 CONAB, Brazil's National Supply Company, is the government agency responsible for providing information on the country's agricultural harvest.
 
-This subset of 27, after inspecting all boundaries in the CONAB public database, appear to be hand-drawn field boundaries.
+These 29 mappings, after inspecting all boundaries in the CONAB public database, appear to be hand-drawn field boundaries.
 
 The content of the Mappings comes from Conab, total or partial reproduction without profit motives is authorized,
 as long as the source is cited and the integrity of the information is maintained.
@@ -78,19 +81,20 @@ class Converter(AdminConverterMixin, FiboaBaseConverter):
 
     def file_migration(self, gdf, path, uri, layer=None):
         gdf = super().file_migration(gdf, path, uri, layer)
-        # Create unique IDs
-        name = Path(path).stem
+        # Create unique IDs, without the upload timestamp the portal prefixes to its file names
+        name = re.sub(r"^\d+_", "", Path(path).stem)
         gdf["id"] = name + "_" + gdf.index.astype(str)
         # Harmonize projection or pd.concat will fail
         if gdf.crs.srs != "EPSG:4674":
             gdf.to_crs(crs="EPSG:4674", inplace=True)
         return gdf
 
     def migrate(self, gdf):
-        gdf = gdf.reset_index(drop=True)
-        gdf["area_ha"] = gdf["area_ha"].combine_first(gdf["Hectares"])
-        gdf["cd_mun"] = gdf["cd_mun"].combine_first(gdf["CD_MUN"]).apply(fformat)
-        gdf["nm_mun"] = gdf["nm_mun"].combine_first(gdf["NM_MUN"]).combine_first(gdf["NM_MUNIC"])
+        # The mappings carry digitising slivers (a fifth of Minas Gerais coffee); no field is that small
+        gdf = gdf[self._measure_area(gdf.geometry) >= MIN_FIELD_AREA].reset_index(drop=True)
+        gdf["area_ha"] = first_of(gdf, "area_ha", "Hectares")
+        gdf["cd_mun"] = first_of(gdf, "cd_mun", "CD_MUN").apply(fformat)
+        gdf["nm_mun"] = first_of(gdf, "nm_mun", "NM_MUN", "NM_MUNIC")
         return super().migrate(gdf)
 
     def get_data(self, paths, **kwargs):
@@ -99,6 +103,15 @@ def get_data(self, paths, **kwargs):
         return super().get_data(paths, **kwargs)
 
 
+def first_of(gdf, *names):
+    """The first non-empty value per row among the columns the files name differently."""
+    result = pd.Series(None, index=gdf.index, dtype=object)
+    for name in names:
+        if name in gdf.columns:
+            result = result.combine_first(gdf[name])
+    return result
+
+
 def fformat(x):
     if isinstance(x, float) and not math.isnan(x):
         return f"{x:.0f}"

tests/test_convert.py

@@ -70,6 +70,7 @@
     "us_ca_scm",
     "us_ca_scm#2018",
     "us_ca_scm#2014",
+    "br_conab",
     "jp",
     "lv",
     "ie",
@@ -165,6 +166,13 @@ def _input_files(converter, *names):
         **_input_files("ch_zh", "ch_zh_2025.gml"),
         "mapping_file": f"{test_path}/ch/lnf_code.csv",
     },
+    # SC names its columns Hectares/NM_MUNIC and is in UTM 22S, RJ uses area_ha/cd_mun in SIRGAS 2000
+    "br_conab": {
+        "input_files": {
+            f"{test_path}/br_conab/1784841407_arroz-sc-safra-2018-2019.zip": ["*.shp"],
+            f"{test_path}/br_conab/1785251708_cafe-rj-21.zip": ["*.shp"],
+        }
+    },
     "es_cl": {
         "variant": "2025",
         "input_files": {f"{test_path}/es_cl/AVILA.zip": ["replaceme.zip"]},
@@ -288,6 +296,11 @@ def _input_files(converter, *names):
     "us_ca_scm#2014": ("U",),
 }
 
+# The smallest field (m²) a converter keeps, where the fixture holds a smaller sliver
+minimum_area = {
+    "br_conab": 100,
+}
+
 # Mapping loaders to patch besides commons.hcat, e.g. where a converter imports one by name
 mapping_lookups = {
     "pt": ("fiboa_cli.datasets.pt.load_hcat_mapping",),
@@ -352,6 +365,10 @@ def load_mapping(csv_file=None, url=None):
             f"e.g. {df.loc[floaty, 'id'].head(3).tolist()}"
         )
 
+    if converter in minimum_area:
+        smallest = df["metrics:area"].min()
+        assert smallest >= minimum_area[converter], f"{converter} kept a {smallest:.1f} m² sliver"
+
     dropped = [c for c in filtered_codes.get(converter, ()) if c in set(df["crop:code"])]
     assert not dropped, f"{converter} kept rows it should filter out: crop:code {dropped}"