US-CA SCM: all nine editions, 2014-2024

CHANGELOG.md

@@ -78,6 +78,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - SI: Extended editions back to 2019.
 - SK: Editions now cover 2018-2026 from the per-campaign datasets on data.slovensko.sk.
 - US-CSB: Editions now cover 2017-2024.
+- US-CA SCM:
+  - Editions now cover 2014, 2016 and 2018-2024 (2024 provisional), from the current portal URLs.
+  - The crop is `MAIN_CROP` from 2019, the main-season `CROPTYP2` in 2016 and 2018, and in 2014 a name mapped to its code through `us_ca_scm_2014.csv`.
+  - The urban mask (`U`, `****`), urban landscape (`UL2`) and riparian vegetation (`NR`) are left out: they are not fields.
+  - The provider is the California Department of Water Resources; `determination:datetime` is 1 July of the edition year.
 
 ### Removed
 - CH: Removed the national `ch` converter, whose single licence could not cover the cantons' differing terms; the canton converters replace it, and a Swiss file is `fiboa merge` of their outputs.

fiboa_cli/datasets/data-files/us_ca_scm_2014.csv

@@ -0,0 +1,48 @@
+original_name,original_code
+Alfalfa and Alfalfa Mixtures,P1
+Almonds,D12
+Apples,D1
+Avocados,C5
+Beans (Dry),F10
+Bush Berries,T19
+Carrots,T6
+Cherries,D3
+Citrus,C
+Cole Crops,T4
+"Corn, Sorghum and Sudan",F16
+Cotton,F1
+Dates,C4
+"Flowers, Nursery and Christmas Tree Farms",T16
+Grapes,V
+Greenhouse,T27
+Idle,X
+Kiwis,C8
+Lettuce/Leafy Greens,T8
+Managed Wetland,NR
+"Melons, Squash and Cucumbers",T9
+Miscellaneous Deciduous,D10
+Miscellaneous Field Crops,F11
+Miscellaneous Grain and Hay,G6
+Miscellaneous Grasses,P6
+Miscellaneous Subtropical Fruits,C7
+Miscellaneous Truck Crops,T18
+Mixed Pasture,P3
+Olives,C6
+Onions and Garlic,T10
+Peaches/Nectarines,D5
+Pears,D6
+Peppers,T21
+Pistachios,D14
+"Plums, Prunes and Apricots",D16
+Pomegranates,D15
+Potatoes and Sweet Potatoes,T31
+Rice,R1
+Safflower,F2
+Strawberries,T20
+Sunflowers,F12
+Tomatoes,T15
+Urban,U
+Walnuts,D13
+Wheat,G2
+Wild Rice,R2
+Young Perennials,YP

fiboa_cli/datasets/us_ca_scm.py

@@ -1,21 +1,64 @@
-from os.path import dirname, join
-
 from vecorel_cli.conversion.admin import AdminConverterMixin
 
 from ..conversion.fiboa_converter import FiboaBaseConverter
-from .commons.hcat import AddHCATMixin, load_hcat_mapping
+from .commons.data import read_data_csv
+from .commons.hcat import AddHCATMixin
+
+BASE = "https://data.cnra.ca.gov/dataset/6c3d65e3-35bb-49e1-a51e-49d5a2cf09a9/resource"
+
+# 2014 names its crops only; the codes are the names' dominant codes in 2016, which has both
+CODES_2014_FILE = "us_ca_scm_2014.csv"
+
+# DWR maps the whole state; these classes are its urban mask (county-sized polygons,
+# "****" in 2020-2022), urban landscape and riparian vegetation, not fields
+NON_AGRICULTURAL = {"U", "****", "UL2", "NR"}
 
 
 class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter):
     variants = {
+        "2024": {
+            f"{BASE}/157956a5-507e-4c6c-b161-b21361488576/download/i15_crop_mapping_2024_provisional_20251208.gdb.zip": [
+                "i15_Crop_Mapping_2024_Provisional_20251208.gdb"
+            ]
+        },
         "2023": {
-            "https://data.cnra.ca.gov/dataset/6c3d65e3-35bb-49e1-a51e-49d5a2cf09a9/resource/4e17ca38-268e-4bf5-bbc5-09636d44ed60/download/i15_crop_mapping_2023_provisional_20241127.gdb.zip": [
-                "i15_Crop_Mapping_2023_Provisional_20241127.gdb"
+            f"{BASE}/4e17ca38-268e-4bf5-bbc5-09636d44ed60/download/i15_crop_mapping_2023_final.gdb.zip": [
+                "i15_Crop_Mapping_2023_Final.gdb"
             ]
         },
         "2022": {
-            "https://data.cnra.ca.gov/dataset/6c3d65e3-35bb-49e1-a51e-49d5a2cf09a9/resource/f38d3f6f-dcf1-4553-9f07-4f381d494320/download/i15_crop_mapping_2022_provisional_gdb.zip": [
-                "i15_Crop_Mapping_2022_Provisional_GDB/i15_Crop_Mapping_2022_Provisional.gdb"
+            f"{BASE}/e41f74d2-7ff9-4871-bc95-1e9673fc53cb/download/i15_crop_mapping_2022.gdb.zip": [
+                "i15_crop_mapping_2022.gdb"
+            ]
+        },
+        "2021": {
+            f"{BASE}/cd1ce211-ac75-44b4-9ea4-345ce2fd0548/download/i15_crop_mapping_2021_gdb.zip": [
+                "i15_Crop_Mapping_2021_GDB/i15_Crop_Mapping_2021.gdb"
+            ]
+        },
+        "2020": {
+            f"{BASE}/44c1bde8-7ac4-4582-b8de-d09264e180fb/download/i15_crop_mapping_2020-gdb.zip": [
+                "i15_Crop_Mapping_2020 GDB/i15_Crop_Mapping_2020.gdb"
+            ]
+        },
+        "2019": {
+            f"{BASE}/519a6ac2-77f5-4da6-85f3-ada74d7eddee/download/i15_crop_mapping_2019_gdb.zip": [
+                "i15_Crop_Mapping_2019_GDB/i15_Crop_Mapping_2019.gdb"
+            ]
+        },
+        "2018": {
+            f"{BASE}/05dc698d-9587-453a-a494-a07beadbbe62/download/i15_crop_mapping_2018_gdb.zip": [
+                "i15_Crop_Mapping_2018_GDB/i15_Crop_Mapping_2018.gdb"
+            ]
+        },
+        "2016": {
+            f"{BASE}/489d7ab8-f68a-45b4-8113-bb89bc4d9a9c/download/i15_crop_mapping_2016_gdb.zip": [
+                "i15_Crop_Mapping_2016_GDB/i15_Crop_Mapping_2016.gdb"
+            ]
+        },
+        "2014": {
+            f"{BASE}/04f89c79-59d1-4981-a3ab-853fdbc79d37/download/i15_crop_mapping_2014_gdb.zip": [
+                "i15_Crop_Mapping_2014_GDB/i15_Crop_Mapping_2014.gdb"
             ]
         },
     }
@@ -26,22 +69,22 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter):
     description = """
 For many years, the California Department of Water Resources (DWR) has collected land use data throughout the state
 and used this information to develop water use estimates for statewide and regional planning efforts, including water
-use projections, water use efficiency evaluation, groundwater model development, and water transfers. These data are
-essential for regional analysis and decision making, which has become increasingly important as DWR and other state agencies
-seek to address resource management issues, regulatory compliance issues, environmental impacts, ecosystem services,
-urban and economic development, and other issues.
+use projections, water use efficiency evaluation, groundwater model development, and water transfers. The statewide
+crop maps are made from remote sensing by Land IQ under contract to DWR. 2024 is provisional.
     """
-    provider = "County of Santa Clara <https://www.sccgov.org/sites/dpd/DocsForms/Documents/Ag_Preserves_Laminated_map.pdf>"
+    provider = "California Department of Water Resources <https://data.cnra.ca.gov/dataset/statewide-crop-mapping>"
+    attribution = "California Department of Water Resources, Statewide Crop Mapping"
     license = "CC0-1.0"
     columns = {
         "geometry": "geometry",
         "UniqueID": "id",
-        "MAIN_CROP": "crop:code",
+        "crop_code": "crop:code",
         "crop:name": "crop:name",
         "COUNTY": "admin_level_2",
+        "determination:datetime": "determination:datetime",
     }
-    column_additions = {
-        "determination:datetime": "2023-05-01T00:00:00Z",
+    column_filters = {
+        "crop_code": lambda col: (col.isin(NON_AGRICULTURAL), True),
     }
     hcat_mapping_csv = "https://fiboa.org/code/us/ca/scm.csv"
     missing_schemas = {
@@ -51,11 +94,17 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter):
     }
 
     def migrate(self, gdf):
-        """
-        Perform migration on the GeoDataFrame to map crop names using a provided mapping file.
-        """
         gdf = super().migrate(gdf)
-        mapping = load_hcat_mapping(url=join(dirname(__file__), "data-files", "us_ca_scm.csv"))
-        original_name_mapping = {e["original_code"]: e["original_name"] for e in mapping}
-        gdf["crop:name"] = gdf["MAIN_CROP"].map(original_name_mapping)
+        # three generations of the schema: names (2014), seasons (2016, 2018), a main crop
+        if "MAIN_CROP" in gdf.columns:
+            gdf["crop_code"] = gdf["MAIN_CROP"]
+        elif "CROPTYP2" in gdf.columns:
+            gdf["crop_code"] = gdf["CROPTYP2"]
+        else:
+            codes = {r["original_name"]: r["original_code"] for r in read_data_csv(CODES_2014_FILE)}
+            gdf["crop_code"] = gdf[f"Crop{self.variant}"].map(codes)
+        gdf["crop:name"] = gdf["crop_code"].map(self.hcat_lookup("original_code", "original_name"))
+        if "County" in gdf.columns:
+            gdf = gdf.rename(columns={"County": "COUNTY"})
+        gdf["determination:datetime"] = f"{self.variant}-07-01T00:00:00Z"
         return gdf

tests/data-files/convert/us_ca_scm/scm.csv

@@ -56,5 +56,26 @@ original_name,original_code,color,translated_name,HCAT3_name,HCAT3_code
 "Vineyards - No Subclass",V,#ebcc65,"Vineyards - No Subclass",vineyards_wine_vine_rebland_grapes,3303060000
 "Not cropped, or unclassified at the time of remote-sensing analysis. Idle status not determined",X,#4debc1,"Not cropped, or unclassified at the time of remote-sensing analysis. Idle status not determined",not_known_and_other,3399000000
 "Young Perennial (grouped for remote sensing or when CLASS C, D or V is not determined)",YP,#ef8c57,"Young Perennial (grouped for remote sensing or when CLASS C, D or V is not determined)",permanent_crops_perennial,3303000000
-"Unclassified",U,#4debc1,"Unclassified",not_known_and_other,3399000000
+"Urban",U,#4debc1,"Urban",not_known_and_other,3399000000
 "Urban Landscape",UL2,#4debc1,"Urban Landscape",not_known_and_other,3399000000
+"Citrus - Lemons",C2,#f1a543,"Citrus - Lemons",citrus_plantations,3303040000
+"Citrus - Oranges",C3,#3abe12,"Citrus - Oranges",citrus_plantations,3303040000
+"Deciduous Fruits and Nuts - No Subclass",D,#f623e7,"Deciduous Fruits and Nuts - No Subclass",permanent_crops_perennial,3303000000
+"Deciduous - Mixed",D11,#c0e3be,"Deciduous - Mixed",unspecified_orchards_fruits,3303019800
+"Plums, Prunes and Apricots (grouped for remote sensing classification only)",D16,#6fd9ec,"Plums, Prunes and Apricots (grouped for remote sensing classification only)",orchards_fruits,3303010000
+"Field Crops - No Subclass",F,#800618,"Field Crops - No Subclass",other_arable_land_crops,3301990000
+"Corn (field and sweet)",F6,#1faf42,"Corn (field and sweet)",grain_maize_corn_popcorn,3301010600
+"Grain and Hay - No Subclass",G,#dfcf28,"Grain and Hay - No Subclass",cereal,3301010000
+"Grain and Hay - Mixed",G7,#bf9175,"Grain and Hay - Mixed",other_arable_land_crops,3301990000
+"Idle - New lands being prepared for crop production",I2,#8213a4,"Idle - New lands being prepared for crop production",fallow_land_not_crop,3301110000
+"Riparian Vegetation",NR,#c3938d,"Riparian Vegetation",not_known_and_other,3399000000
+"Pasture - No Subclass",P,#44c29e,"Pasture - No Subclass",pasture_meadow_grassland_grass,3302000000
+"Pasture - Clover",P2,#589059,"Pasture - Clover",clover,3301090303
+"Rice - No Subclass",R,#e1e1d3,"Rice - No Subclass",rice,3301010700
+"Truck, Nursery and Berry Crops - No Subclass",T,#b9ece1,"Truck, Nursery and Berry Crops - No Subclass",fresh_vegetables,3301070000
+"Artichokes",T1,#ce499d,"Artichokes",artichoke,3301270000
+"Tomatoes (processing)",T15,#b6520b,"Tomatoes (processing)",tomato,3301280000
+"Truck Crops - Mixed (four or more)",T17,#cc75c1,"Truck Crops - Mixed (four or more)",fresh_vegetables,3301070000
+"Potato and Sweet Potato (grouped for remote sensing classification only)",T31,#25ddda,"Potato and Sweet Potato (grouped for remote sensing classification only)",potatoes,3301030000
+"Lettuce (all types)",T8,#001cf4,"Lettuce (all types)",salads_lettuce_leaf_vegetables,3301310000
+"Vineyards - Wine Grapes",V2,#ec765f,"Vineyards - Wine Grapes",vineyards_wine_vine_rebland_grapes,3303060000

tests/test_convert.py

@@ -68,6 +68,8 @@
     "cz#2020",
     "us_usda_cropland",
     "us_ca_scm",
+    "us_ca_scm#2018",
+    "us_ca_scm#2014",
     "jp",
     "lv",
     "ie",
@@ -177,6 +179,10 @@ def _input_files(converter, *names):
         "input_files": {f"{test_path}/es_an/SP25_REC_PROV_04.zip": ["SP25_REC_04.shp"]},
     },
     "sk#2018": {"variant": "2018"},
+    # three schema generations: a main crop, crops per season (2016, 2018), crop names (2014)
+    "us_ca_scm": {"variant": "2024"},
+    "us_ca_scm#2018": {"variant": "2018"},
+    "us_ca_scm#2014": {"variant": "2014"},
     "bg": {"variant": "2025", **_input_files("bg", "bg_agricultural_land_2025.zip")},
     "bg#2022": {"variant": "2022", **_input_files("bg", "bg_agricultural_land_2022.zip")},
     "es_cat": _input_files("es_cat", "Cultius_DUN2023_GPKG.zip"),
@@ -275,6 +281,13 @@ def _input_files(converter, *names):
     },
 }
 
+# Classes a converter filters out, and that the fixture does contain
+filtered_codes = {
+    # DWR maps the whole state: the urban mask is county-sized polygons, not fields
+    "us_ca_scm#2018": ("U",),
+    "us_ca_scm#2014": ("U",),
+}
+
 # Mapping loaders to patch besides commons.hcat, e.g. where a converter imports one by name
 mapping_lookups = {
     "pt": ("fiboa_cli.datasets.pt.load_hcat_mapping",),
@@ -339,6 +352,9 @@ def load_mapping(csv_file=None, url=None):
             f"e.g. {df.loc[floaty, 'id'].head(3).tolist()}"
         )
 
+    dropped = [c for c in filtered_codes.get(converter, ()) if c in set(df["crop:code"])]
+    assert not dropped, f"{converter} kept rows it should filter out: crop:code {dropped}"
+
     if "metrics:area" in df.columns and converter not in ("de_bb",):
         # Check for accidental hectare conversion; fields should be more than 10 square meters
         assert (df["metrics:area"] > 10).all()