diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d4d3b74..15d8c24f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - SI: Extended editions back to 2019. - SK: Editions now cover 2018-2026 from the per-campaign datasets on data.slovensko.sk. - US-CSB: Editions now cover 2017-2024. +- US-CA SCM: + - Editions now cover 2014, 2016 and 2018-2024 (2024 provisional), from the current portal URLs. + - The crop is `MAIN_CROP` from 2019, the main-season `CROPTYP2` in 2016 and 2018, and in 2014 a name mapped to its code through `us_ca_scm_2014.csv`. + - The urban mask (`U`, `****`), urban landscape (`UL2`) and riparian vegetation (`NR`) are left out: they are not fields. + - The provider is the California Department of Water Resources; `determination:datetime` is 1 July of the edition year. ### Removed - CH: Removed the national `ch` converter, whose single licence could not cover the cantons' differing terms; the canton converters replace it, and a Swiss file is `fiboa merge` of their outputs. diff --git a/fiboa_cli/datasets/data-files/us_ca_scm.csv b/fiboa_cli/datasets/data-files/us_ca_scm.csv deleted file mode 100644 index c332f615..00000000 --- a/fiboa_cli/datasets/data-files/us_ca_scm.csv +++ /dev/null @@ -1,58 +0,0 @@ -original_name,original_code,color -"****",****,#44aae6 -"Citrus and Subtropical - No Subclass",C,#cb1469 -"Eucalyptus",C10,#410cc9 -"Dates",C4,#43ce34 -"Avocados",C5,#86cb3c -"Olives",C6,#d84d72 -"Subtropical Fruits Misc.",C7,#5eda3f -"Kiwis",C8,#d18274 -"Apples",D1,#d4886f -"Deciduous - Misc.",D10,#9f54d1 -"Almonds",D12,#dfad84 -"Walnuts",D13,#665aea -"Pistachios",D14,#b063d1 -"Pomegranates",D15,#cf8429 -"Pecans",D17,#0fe769 -"Apricots",D2,#1546e7 -"Cherries",D3,#31e4a2 -"Peaches and Nectarines",D5,#b5db77 -"Pears",D6,#ba84ef -"Plums",D7,#ebe41d -"Prunes",D8,#6649e9 -"Cotton",F1,#6ee185 -"Beans (dry)",F10,#e4777e -"Field Misc.",F11,#dcea13 -"Sunflowers",F12,#86d3ec -"Corn, Sorghum or Sudan (grouped for remote sensing classification only)",F16,#0e55e3 -"Safflower",F2,#26cd5b -"Sugar beets",F5,#b5d32f -"Wheat",G2,#7ad3cf -"Grain and Hay - Misc.",G6,#69cdbe -"Idle - Land not cropped in current or prior year, but within last 3 yrs.",I1,#1e38e1 -"Idle - Long Term - land consistently idle for four or more years",I4,#6aade8 -"Alfalfa and alfalfa mixtures",P1,#e1b96a -"Pasture - Mixed",P3,#7a3cde -"Pasture - Native Improved",P4,#de3853 -"Pasture - Induced High Water",P5,#ed2d26 -"Pasture - Miscellaneous Grasses",P6,#61eb26 -"Pasture - Turf Farms",P7,#5b99e9 -"Rice",R1,#c872b5 -"Rice - Wild",R2,#f035a2 -"Onions and Garlic",T10,#4eec5e -"Potatoes",T12,#e7da85 -"Sweet Potatoes",T13,#ef6ec6 -"Flowers, nursery and Christmas Tree Farms",T16,#6264e5 -"Truck Crops - Misc.",T18,#e95ad8 -"Bushberries",T19,#96d26c -"Strawberries",T20,#d116e6 -"Peppers (Chili, Bell, etc.)",T21,#73cbda -"Greenhouse",T27,#ab1ccf -"Lettuce or Leafy Greens (grouped for remote sensing classification only)",T30,#47d047 -"Tomatoes (all)",T32,#de4980 -"Cole crops (mixture of T22-T25)",T4,#31db89 -"Carrots",T6,#cb19c8 -"Melons, Squash, and Cucumbers",T9,#bdd879 -"Vineyards - No Subclass",V,#ebcc65 -"Not cropped, or unclassified at the time of remote-sensing analysis. Idle status not determined",X,#4debc1 -"Young Perennial (grouped for remote sensing or when CLASS C, D or V is not determined)",YP,#ef8c57 diff --git a/fiboa_cli/datasets/data-files/us_ca_scm_2014.csv b/fiboa_cli/datasets/data-files/us_ca_scm_2014.csv new file mode 100644 index 00000000..be774d53 --- /dev/null +++ b/fiboa_cli/datasets/data-files/us_ca_scm_2014.csv @@ -0,0 +1,48 @@ +original_name,original_code +Alfalfa and Alfalfa Mixtures,P1 +Almonds,D12 +Apples,D1 +Avocados,C5 +Beans (Dry),F10 +Bush Berries,T19 +Carrots,T6 +Cherries,D3 +Citrus,C +Cole Crops,T4 +"Corn, Sorghum and Sudan",F16 +Cotton,F1 +Dates,C4 +"Flowers, Nursery and Christmas Tree Farms",T16 +Grapes,V +Greenhouse,T27 +Idle,X +Kiwis,C8 +Lettuce/Leafy Greens,T8 +Managed Wetland,NR +"Melons, Squash and Cucumbers",T9 +Miscellaneous Deciduous,D10 +Miscellaneous Field Crops,F11 +Miscellaneous Grain and Hay,G6 +Miscellaneous Grasses,P6 +Miscellaneous Subtropical Fruits,C7 +Miscellaneous Truck Crops,T18 +Mixed Pasture,P3 +Olives,C6 +Onions and Garlic,T10 +Peaches/Nectarines,D5 +Pears,D6 +Peppers,T21 +Pistachios,D14 +"Plums, Prunes and Apricots",D16 +Pomegranates,D15 +Potatoes and Sweet Potatoes,T31 +Rice,R1 +Safflower,F2 +Strawberries,T20 +Sunflowers,F12 +Tomatoes,T15 +Urban,U +Walnuts,D13 +Wheat,G2 +Wild Rice,R2 +Young Perennials,YP diff --git a/fiboa_cli/datasets/us_ca_scm.py b/fiboa_cli/datasets/us_ca_scm.py index 69cf7f53..bcc6609c 100644 --- a/fiboa_cli/datasets/us_ca_scm.py +++ b/fiboa_cli/datasets/us_ca_scm.py @@ -1,21 +1,64 @@ -from os.path import dirname, join - from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter -from .commons.hcat import AddHCATMixin, load_hcat_mapping +from .commons.data import read_data_csv +from .commons.hcat import AddHCATMixin + +BASE = "https://data.cnra.ca.gov/dataset/6c3d65e3-35bb-49e1-a51e-49d5a2cf09a9/resource" + +# 2014 names its crops only; the codes are the names' dominant codes in 2016, which has both +CODES_2014_FILE = "us_ca_scm_2014.csv" + +# DWR maps the whole state; these classes are its urban mask (county-sized polygons, +# "****" in 2020-2022), urban landscape and riparian vegetation, not fields +NON_AGRICULTURAL = {"U", "****", "UL2", "NR"} class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): variants = { + "2024": { + f"{BASE}/157956a5-507e-4c6c-b161-b21361488576/download/i15_crop_mapping_2024_provisional_20251208.gdb.zip": [ + "i15_Crop_Mapping_2024_Provisional_20251208.gdb" + ] + }, "2023": { - "https://data.cnra.ca.gov/dataset/6c3d65e3-35bb-49e1-a51e-49d5a2cf09a9/resource/4e17ca38-268e-4bf5-bbc5-09636d44ed60/download/i15_crop_mapping_2023_provisional_20241127.gdb.zip": [ - "i15_Crop_Mapping_2023_Provisional_20241127.gdb" + f"{BASE}/4e17ca38-268e-4bf5-bbc5-09636d44ed60/download/i15_crop_mapping_2023_final.gdb.zip": [ + "i15_Crop_Mapping_2023_Final.gdb" ] }, "2022": { - "https://data.cnra.ca.gov/dataset/6c3d65e3-35bb-49e1-a51e-49d5a2cf09a9/resource/f38d3f6f-dcf1-4553-9f07-4f381d494320/download/i15_crop_mapping_2022_provisional_gdb.zip": [ - "i15_Crop_Mapping_2022_Provisional_GDB/i15_Crop_Mapping_2022_Provisional.gdb" + f"{BASE}/e41f74d2-7ff9-4871-bc95-1e9673fc53cb/download/i15_crop_mapping_2022.gdb.zip": [ + "i15_crop_mapping_2022.gdb" + ] + }, + "2021": { + f"{BASE}/cd1ce211-ac75-44b4-9ea4-345ce2fd0548/download/i15_crop_mapping_2021_gdb.zip": [ + "i15_Crop_Mapping_2021_GDB/i15_Crop_Mapping_2021.gdb" + ] + }, + "2020": { + f"{BASE}/44c1bde8-7ac4-4582-b8de-d09264e180fb/download/i15_crop_mapping_2020-gdb.zip": [ + "i15_Crop_Mapping_2020 GDB/i15_Crop_Mapping_2020.gdb" + ] + }, + "2019": { + f"{BASE}/519a6ac2-77f5-4da6-85f3-ada74d7eddee/download/i15_crop_mapping_2019_gdb.zip": [ + "i15_Crop_Mapping_2019_GDB/i15_Crop_Mapping_2019.gdb" + ] + }, + "2018": { + f"{BASE}/05dc698d-9587-453a-a494-a07beadbbe62/download/i15_crop_mapping_2018_gdb.zip": [ + "i15_Crop_Mapping_2018_GDB/i15_Crop_Mapping_2018.gdb" + ] + }, + "2016": { + f"{BASE}/489d7ab8-f68a-45b4-8113-bb89bc4d9a9c/download/i15_crop_mapping_2016_gdb.zip": [ + "i15_Crop_Mapping_2016_GDB/i15_Crop_Mapping_2016.gdb" + ] + }, + "2014": { + f"{BASE}/04f89c79-59d1-4981-a3ab-853fdbc79d37/download/i15_crop_mapping_2014_gdb.zip": [ + "i15_Crop_Mapping_2014_GDB/i15_Crop_Mapping_2014.gdb" ] }, } @@ -26,22 +69,22 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): description = """ For many years, the California Department of Water Resources (DWR) has collected land use data throughout the state and used this information to develop water use estimates for statewide and regional planning efforts, including water -use projections, water use efficiency evaluation, groundwater model development, and water transfers. These data are -essential for regional analysis and decision making, which has become increasingly important as DWR and other state agencies -seek to address resource management issues, regulatory compliance issues, environmental impacts, ecosystem services, -urban and economic development, and other issues. +use projections, water use efficiency evaluation, groundwater model development, and water transfers. The statewide +crop maps are made from remote sensing by Land IQ under contract to DWR. 2024 is provisional. """ - provider = "County of Santa Clara " + provider = "California Department of Water Resources " + attribution = "California Department of Water Resources, Statewide Crop Mapping" license = "CC0-1.0" columns = { "geometry": "geometry", "UniqueID": "id", - "MAIN_CROP": "crop:code", + "crop_code": "crop:code", "crop:name": "crop:name", "COUNTY": "admin_level_2", + "determination:datetime": "determination:datetime", } - column_additions = { - "determination:datetime": "2023-05-01T00:00:00Z", + column_filters = { + "crop_code": lambda col: (col.isin(NON_AGRICULTURAL), True), } hcat_mapping_csv = "https://fiboa.org/code/us/ca/scm.csv" missing_schemas = { @@ -51,11 +94,17 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): } def migrate(self, gdf): - """ - Perform migration on the GeoDataFrame to map crop names using a provided mapping file. - """ gdf = super().migrate(gdf) - mapping = load_hcat_mapping(url=join(dirname(__file__), "data-files", "us_ca_scm.csv")) - original_name_mapping = {e["original_code"]: e["original_name"] for e in mapping} - gdf["crop:name"] = gdf["MAIN_CROP"].map(original_name_mapping) + # three generations of the schema: names (2014), seasons (2016, 2018), a main crop + if "MAIN_CROP" in gdf.columns: + gdf["crop_code"] = gdf["MAIN_CROP"] + elif "CROPTYP2" in gdf.columns: + gdf["crop_code"] = gdf["CROPTYP2"] + else: + codes = {r["original_name"]: r["original_code"] for r in read_data_csv(CODES_2014_FILE)} + gdf["crop_code"] = gdf[f"Crop{self.variant}"].map(codes) + gdf["crop:name"] = gdf["crop_code"].map(self.hcat_lookup("original_code", "original_name")) + if "County" in gdf.columns: + gdf = gdf.rename(columns={"County": "COUNTY"}) + gdf["determination:datetime"] = f"{self.variant}-07-01T00:00:00Z" return gdf diff --git a/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2014_gdb.zip b/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2014_gdb.zip new file mode 100644 index 00000000..e0250676 Binary files /dev/null and b/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2014_gdb.zip differ diff --git a/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2018_gdb.zip b/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2018_gdb.zip new file mode 100644 index 00000000..a561972d Binary files /dev/null and b/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2018_gdb.zip differ diff --git a/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2023_provisional_20241127.gdb.zip b/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2023_provisional_20241127.gdb.zip deleted file mode 100644 index 0b662e1e..00000000 Binary files a/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2023_provisional_20241127.gdb.zip and /dev/null differ diff --git a/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2024_provisional_20251208.gdb.zip b/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2024_provisional_20251208.gdb.zip new file mode 100644 index 00000000..9fac9d17 Binary files /dev/null and b/tests/data-files/convert/us_ca_scm/i15_crop_mapping_2024_provisional_20251208.gdb.zip differ diff --git a/tests/data-files/convert/us_ca_scm/scm.csv b/tests/data-files/convert/us_ca_scm/scm.csv index 87bdf1ff..f86914d9 100644 --- a/tests/data-files/convert/us_ca_scm/scm.csv +++ b/tests/data-files/convert/us_ca_scm/scm.csv @@ -56,5 +56,26 @@ original_name,original_code,color,translated_name,HCAT3_name,HCAT3_code "Vineyards - No Subclass",V,#ebcc65,"Vineyards - No Subclass",vineyards_wine_vine_rebland_grapes,3303060000 "Not cropped, or unclassified at the time of remote-sensing analysis. Idle status not determined",X,#4debc1,"Not cropped, or unclassified at the time of remote-sensing analysis. Idle status not determined",not_known_and_other,3399000000 "Young Perennial (grouped for remote sensing or when CLASS C, D or V is not determined)",YP,#ef8c57,"Young Perennial (grouped for remote sensing or when CLASS C, D or V is not determined)",permanent_crops_perennial,3303000000 -"Unclassified",U,#4debc1,"Unclassified",not_known_and_other,3399000000 +"Urban",U,#4debc1,"Urban",not_known_and_other,3399000000 "Urban Landscape",UL2,#4debc1,"Urban Landscape",not_known_and_other,3399000000 +"Citrus - Lemons",C2,#f1a543,"Citrus - Lemons",citrus_plantations,3303040000 +"Citrus - Oranges",C3,#3abe12,"Citrus - Oranges",citrus_plantations,3303040000 +"Deciduous Fruits and Nuts - No Subclass",D,#f623e7,"Deciduous Fruits and Nuts - No Subclass",permanent_crops_perennial,3303000000 +"Deciduous - Mixed",D11,#c0e3be,"Deciduous - Mixed",unspecified_orchards_fruits,3303019800 +"Plums, Prunes and Apricots (grouped for remote sensing classification only)",D16,#6fd9ec,"Plums, Prunes and Apricots (grouped for remote sensing classification only)",orchards_fruits,3303010000 +"Field Crops - No Subclass",F,#800618,"Field Crops - No Subclass",other_arable_land_crops,3301990000 +"Corn (field and sweet)",F6,#1faf42,"Corn (field and sweet)",grain_maize_corn_popcorn,3301010600 +"Grain and Hay - No Subclass",G,#dfcf28,"Grain and Hay - No Subclass",cereal,3301010000 +"Grain and Hay - Mixed",G7,#bf9175,"Grain and Hay - Mixed",other_arable_land_crops,3301990000 +"Idle - New lands being prepared for crop production",I2,#8213a4,"Idle - New lands being prepared for crop production",fallow_land_not_crop,3301110000 +"Riparian Vegetation",NR,#c3938d,"Riparian Vegetation",not_known_and_other,3399000000 +"Pasture - No Subclass",P,#44c29e,"Pasture - No Subclass",pasture_meadow_grassland_grass,3302000000 +"Pasture - Clover",P2,#589059,"Pasture - Clover",clover,3301090303 +"Rice - No Subclass",R,#e1e1d3,"Rice - No Subclass",rice,3301010700 +"Truck, Nursery and Berry Crops - No Subclass",T,#b9ece1,"Truck, Nursery and Berry Crops - No Subclass",fresh_vegetables,3301070000 +"Artichokes",T1,#ce499d,"Artichokes",artichoke,3301270000 +"Tomatoes (processing)",T15,#b6520b,"Tomatoes (processing)",tomato,3301280000 +"Truck Crops - Mixed (four or more)",T17,#cc75c1,"Truck Crops - Mixed (four or more)",fresh_vegetables,3301070000 +"Potato and Sweet Potato (grouped for remote sensing classification only)",T31,#25ddda,"Potato and Sweet Potato (grouped for remote sensing classification only)",potatoes,3301030000 +"Lettuce (all types)",T8,#001cf4,"Lettuce (all types)",salads_lettuce_leaf_vegetables,3301310000 +"Vineyards - Wine Grapes",V2,#ec765f,"Vineyards - Wine Grapes",vineyards_wine_vine_rebland_grapes,3303060000 diff --git a/tests/test_convert.py b/tests/test_convert.py index 71152081..1b7ff944 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -68,6 +68,8 @@ "cz#2020", "us_usda_cropland", "us_ca_scm", + "us_ca_scm#2018", + "us_ca_scm#2014", "jp", "lv", "ie", @@ -177,6 +179,10 @@ def _input_files(converter, *names): "input_files": {f"{test_path}/es_an/SP25_REC_PROV_04.zip": ["SP25_REC_04.shp"]}, }, "sk#2018": {"variant": "2018"}, + # three schema generations: a main crop, crops per season (2016, 2018), crop names (2014) + "us_ca_scm": {"variant": "2024"}, + "us_ca_scm#2018": {"variant": "2018"}, + "us_ca_scm#2014": {"variant": "2014"}, "bg": {"variant": "2025", **_input_files("bg", "bg_agricultural_land_2025.zip")}, "bg#2022": {"variant": "2022", **_input_files("bg", "bg_agricultural_land_2022.zip")}, "es_cat": _input_files("es_cat", "Cultius_DUN2023_GPKG.zip"), @@ -275,6 +281,13 @@ def _input_files(converter, *names): }, } +# Classes a converter filters out, and that the fixture does contain +filtered_codes = { + # DWR maps the whole state: the urban mask is county-sized polygons, not fields + "us_ca_scm#2018": ("U",), + "us_ca_scm#2014": ("U",), +} + # Mapping loaders to patch besides commons.hcat, e.g. where a converter imports one by name mapping_lookups = { "pt": ("fiboa_cli.datasets.pt.load_hcat_mapping",), @@ -339,6 +352,9 @@ def load_mapping(csv_file=None, url=None): f"e.g. {df.loc[floaty, 'id'].head(3).tolist()}" ) + dropped = [c for c in filtered_codes.get(converter, ()) if c in set(df["crop:code"])] + assert not dropped, f"{converter} kept rows it should filter out: crop:code {dropped}" + if "metrics:area" in df.columns and converter not in ("de_bb",): # Check for accidental hectare conversion; fields should be more than 10 square meters assert (df["metrics:area"] > 10).all()