diff --git a/CHANGELOG.md b/CHANGELOG.md index d71fffd3..6d061dab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Converters now publish `metrics:area` by default (`area_calculate_missing = True`, #277): measured from the geometry for every row if the source has no area, and for the rows where it is empty or 0 otherwise. Set `area_calculate_missing = False` to opt out. - Renamed the HCAT mapping attributes of `AddHCATMixin`, as they work with any HCAT mapping, not only EuroCrops: the `ec_mapping*` attributes use the `hcat_mapping*` prefix now. The helpers `load_ec_mapping()` and `ec_url()` are now `load_hcat_mapping()` and `hcat_mapping_url()` in `fiboa_cli.datasets.commons.hcat`; the copies in `fiboa_cli.datasets.commons.ec` were removed. - BE-VLG: Extended editions to 2018-2026 and aligned determination dates with the selected campaign year. +- BR-CONAB: Reads the 29 mappings from the portal's new download URLs; ids no longer carry the file's upload timestamp, and polygons under 100 m² (digitising slivers, a fifth of Minas Gerais coffee) are left out. - CZ: Extended year coverage, including GPZ_DP editions (2019-2022), and added 2026 nested-archive support. - DE-SH: Extended support to editions 2023, 2025 and 2026. - DK: Editions now cover 2008-2026. The 2008 and 2009 editions are published without the crop and HCAT extensions because the source has no crop columns. diff --git a/fiboa_cli/datasets/br_conab.py b/fiboa_cli/datasets/br_conab.py index c03a86ed..5d433ca3 100644 --- a/fiboa_cli/datasets/br_conab.py +++ b/fiboa_cli/datasets/br_conab.py @@ -1,44 +1,47 @@ import math +import re from pathlib import Path +import pandas as pd from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter +MIN_FIELD_AREA = 100 # m² + class Converter(AdminConverterMixin, FiboaBaseConverter): + # The portal's file list: https://barramento.conab.gov.br/portal-informacao-api/api/v1/download _sources = [ - "Algodao/GO/ALGODAO_GO_Safra_2019_2020.zip", - "Algodao/GO/ALGODAO-GO_Safra_2018_2019.zip", - "Algodao/GO/GO_ALGODAO_2021.zip", - "Algodao/GO/GO_ALGODAO_2223.zip", - "Algodao/MS/MS_ALGODAO_2021.zip", - "Algodao/MS/MS_ALGODAO_2122.zip", - "Arroz_Irrigado/GO/GO_ARROZ_IRRIG_2122.zip", - "Arroz_Irrigado/GO/GO_ARROZ_IRRIG_INUND_2324.zip", - "Arroz_Irrigado/GO/ARROZ-GO_Safra_2018_2019.zip", - "Arroz_Irrigado/MS/ARROZ-MS_Safra_2018_2019.zip", - "Arroz_Irrigado/PR/ARROZ-PR_Safra_2017_2018.zip", - "Arroz_Irrigado/RS/ARROZ-RS_Safra_2019_2020.zip", - "Arroz_Irrigado/SC/ARROZ-SC_Safra_2018_2019.zip", - "Arroz_Irrigado/TO/ARROZ-TO_Safra_2017_2018.zip", - "Arroz_Irrigado/GO/GO_ARROZ_IRRIG_INUND_2324.zip", - "Arroz_Irrigado/TO/TO_ARROZ_IRRIG_2324.zip", - "Cana/GO/CANA-GO_Safra_2011_2012.zip", - "Cafe/BA/CAFE-BA_Safra_2019.zip", - "Cafe/DF/CAFE-DF_Safra_2018.zip", - "Cafe/GO/CAFE-GO_Safra_2018.zip", - "Cafe/GO/CAFE-GO_Safra_2019.zip", - "Cafe/PR/CAFE-PR_Safra_2017.zip", - "Cafe/MG/CAFE-MG_Safra_2017.zip", - "Cafe/DF/DF_CAFE_24.zip", - "Cafe/DF/DF_CAFE_24.zip", - "Cafe/GO/GO_CAFE_21.zip", - "Cafe/RJ/RJ_CAFE_21.zip", - "Culturas_de_Verao_1_Safra/DF/CV-DF_Safra_2013_2014.zip", - "Culturas_de_Verao_1_Safra/DF/CV-DF_Safra_2014_2015.zip", - "Culturas_de_Verao_1_Safra/DF/CV-DF_Safra_2017_2018.zip", - "Culturas_de_Verao_1_Safra/TO/CV-TO_Safra_2019_2020.zip", + "1784839922_algodao-go-safra-2019-2020.zip", + "1784839955_algodao-go-safra-2018-2019.zip", + "1784839890_go-algodao-2021.zip", + "1784558283_go-algodao-2223.zip", + "1784840252_ms-algodao-2021.zip", + "1784840209_ms-algodao-2122.zip", + "1784840479_go-arroz-irrig-2122.zip", + "1784840507_go-arroz-irrig-inund-23241.zip", + "1784840800_arroz-go-safra-2018-2019.zip", + "1784840902_arroz-ms-safra-2018-2019.zip", + "1784841005_arroz-pr-safra-2017-2018.zip", + "1784841189_arroz-rs-safra-2019-2020.zip", + "1784841407_arroz-sc-safra-2018-2019.zip", + "1784841526_arroz-to-safra-2017-2018.zip", + "1784841496_to-arroz-irrig-2324.zip", + "1785258893_cana-go-11-12.zip", + "1789672175_cafe-ba-19.zip", + "1785251263_cafe-df-18.zip", + "1785251314_cafe-go-18.zip", + "1785251334_cafe-go-19.zip", + "1785251366_cafe-pr-17.zip", + "1787760927_cafe-mg-safra-2017.zip", + "1785251294_cafe-df-24.zip", + "1785251350_cafe-go-21.zip", + "1785251708_cafe-rj-21.zip", + "1785335351_cv-df-safra-2013-2014.zip", + "1785335351_cv-df-safra-2014-2015.zip", + "1785335352_cv-df-safra-2017-2018.zip", + "1785335410_cv-to-safra-2019-2020.zip", ] sources = { "https://portaldeinformacoes.conab.gov.br/downloads/mapas/" + k: ["*.shp"] for k in _sources @@ -49,7 +52,7 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): description = """ CONAB, Brazil's National Supply Company, is the government agency responsible for providing information on the country's agricultural harvest. -This subset of 27, after inspecting all boundaries in the CONAB public database, appear to be hand-drawn field boundaries. +These 29 mappings, after inspecting all boundaries in the CONAB public database, appear to be hand-drawn field boundaries. The content of the Mappings comes from Conab, total or partial reproduction without profit motives is authorized, as long as the source is cited and the integrity of the information is maintained. @@ -78,8 +81,8 @@ class Converter(AdminConverterMixin, FiboaBaseConverter): def file_migration(self, gdf, path, uri, layer=None): gdf = super().file_migration(gdf, path, uri, layer) - # Create unique IDs - name = Path(path).stem + # Create unique IDs, without the upload timestamp the portal prefixes to its file names + name = re.sub(r"^\d+_", "", Path(path).stem) gdf["id"] = name + "_" + gdf.index.astype(str) # Harmonize projection or pd.concat will fail if gdf.crs.srs != "EPSG:4674": @@ -87,10 +90,11 @@ def file_migration(self, gdf, path, uri, layer=None): return gdf def migrate(self, gdf): - gdf = gdf.reset_index(drop=True) - gdf["area_ha"] = gdf["area_ha"].combine_first(gdf["Hectares"]) - gdf["cd_mun"] = gdf["cd_mun"].combine_first(gdf["CD_MUN"]).apply(fformat) - gdf["nm_mun"] = gdf["nm_mun"].combine_first(gdf["NM_MUN"]).combine_first(gdf["NM_MUNIC"]) + # The mappings carry digitising slivers (a fifth of Minas Gerais coffee); no field is that small + gdf = gdf[self._measure_area(gdf.geometry) >= MIN_FIELD_AREA].reset_index(drop=True) + gdf["area_ha"] = first_of(gdf, "area_ha", "Hectares") + gdf["cd_mun"] = first_of(gdf, "cd_mun", "CD_MUN").apply(fformat) + gdf["nm_mun"] = first_of(gdf, "nm_mun", "NM_MUN", "NM_MUNIC") return super().migrate(gdf) def get_data(self, paths, **kwargs): @@ -99,6 +103,15 @@ def get_data(self, paths, **kwargs): return super().get_data(paths, **kwargs) +def first_of(gdf, *names): + """The first non-empty value per row among the columns the files name differently.""" + result = pd.Series(None, index=gdf.index, dtype=object) + for name in names: + if name in gdf.columns: + result = result.combine_first(gdf[name]) + return result + + def fformat(x): if isinstance(x, float) and not math.isnan(x): return f"{x:.0f}" diff --git a/tests/data-files/convert/br_conab/1784841407_arroz-sc-safra-2018-2019.zip b/tests/data-files/convert/br_conab/1784841407_arroz-sc-safra-2018-2019.zip new file mode 100644 index 00000000..9a3929df Binary files /dev/null and b/tests/data-files/convert/br_conab/1784841407_arroz-sc-safra-2018-2019.zip differ diff --git a/tests/data-files/convert/br_conab/1785251708_cafe-rj-21.zip b/tests/data-files/convert/br_conab/1785251708_cafe-rj-21.zip new file mode 100644 index 00000000..01acc97c Binary files /dev/null and b/tests/data-files/convert/br_conab/1785251708_cafe-rj-21.zip differ diff --git a/tests/test_convert.py b/tests/test_convert.py index 1b7ff944..14fe66ad 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -70,6 +70,7 @@ "us_ca_scm", "us_ca_scm#2018", "us_ca_scm#2014", + "br_conab", "jp", "lv", "ie", @@ -165,6 +166,13 @@ def _input_files(converter, *names): **_input_files("ch_zh", "ch_zh_2025.gml"), "mapping_file": f"{test_path}/ch/lnf_code.csv", }, + # SC names its columns Hectares/NM_MUNIC and is in UTM 22S, RJ uses area_ha/cd_mun in SIRGAS 2000 + "br_conab": { + "input_files": { + f"{test_path}/br_conab/1784841407_arroz-sc-safra-2018-2019.zip": ["*.shp"], + f"{test_path}/br_conab/1785251708_cafe-rj-21.zip": ["*.shp"], + } + }, "es_cl": { "variant": "2025", "input_files": {f"{test_path}/es_cl/AVILA.zip": ["replaceme.zip"]}, @@ -288,6 +296,11 @@ def _input_files(converter, *names): "us_ca_scm#2014": ("U",), } +# The smallest field (m²) a converter keeps, where the fixture holds a smaller sliver +minimum_area = { + "br_conab": 100, +} + # Mapping loaders to patch besides commons.hcat, e.g. where a converter imports one by name mapping_lookups = { "pt": ("fiboa_cli.datasets.pt.load_hcat_mapping",), @@ -352,6 +365,10 @@ def load_mapping(csv_file=None, url=None): f"e.g. {df.loc[floaty, 'id'].head(3).tolist()}" ) + if converter in minimum_area: + smallest = df["metrics:area"].min() + assert smallest >= minimum_area[converter], f"{converter} kept a {smallest:.1f} m² sliver" + dropped = [c for c in filtered_codes.get(converter, ()) if c in set(df["crop:code"])] assert not dropped, f"{converter} kept rows it should filter out: crop:code {dropped}"