From e3e01846d1a07f80d4f916c6c65d09be23b91760 Mon Sep 17 00:00:00 2001 From: David del Real Sifuentes Date: Mon, 28 Sep 2026 18:00:55 +0000 Subject: [PATCH] chore(composer): remove obsolete composer data orchestration sample Originally reviewed to resolve Dependabot alerts, this sample is being removed because it relies on unmaintained pre-built containers and obsolete dependencies, and is no longer referenced on the documentation site. This sample has been superseded by the new RunInference approach. For the current documentation and code, see: https://docs.cloud.google.com/dataflow/docs/notebooks/run_inference_huggingface?utm_source=gemini --- .../README.md | 3 - .../__init__.py | 0 .../conftest.py | 40 -- .../constraints.txt | 679 ------------------ .../dags/__init__.py | 0 .../data_orchestration_blog_sample_dag.py | 97 --- ...data_orchestration_blog_sample_dag_test.py | 40 -- .../noxfile_config.py | 58 -- .../requirements-test.txt | 2 - .../requirements.txt | 6 - 10 files changed, 925 deletions(-) delete mode 100644 composer/blog/gcp-tech-blog/data-orchestration-with-composer/README.md delete mode 100644 composer/blog/gcp-tech-blog/data-orchestration-with-composer/__init__.py delete mode 100644 composer/blog/gcp-tech-blog/data-orchestration-with-composer/conftest.py delete mode 100644 composer/blog/gcp-tech-blog/data-orchestration-with-composer/constraints.txt delete mode 100644 composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/__init__.py delete mode 100644 composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag.py delete mode 100644 composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag_test.py delete mode 100644 composer/blog/gcp-tech-blog/data-orchestration-with-composer/noxfile_config.py delete mode 100644 composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements-test.txt delete mode 100644 composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements.txt diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/README.md b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/README.md deleted file mode 100644 index 7d0e250dd81..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Data Orchestration with Cloud Composer - -Code found in this directory is part of a [blog post](https://cloud.google.com/blog/topics/developers-practitioners) published in June 2021 \ No newline at end of file diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/__init__.py b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/conftest.py b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/conftest.py deleted file mode 100644 index e7033247004..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/conftest.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2021 Google LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# https://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import os - -import pytest - - -# this fixture initializes the Airflow DB once per session -# it is used by DAGs in both the blogs and workflows directories, -# unless there exists a conftest at a lower level -@pytest.fixture(scope="session") -def airflow_database(): - import airflow.utils.db - - # We use separate directory for local db path per session - # by setting AIRFLOW_HOME env var, which is done in noxfile_config.py. - - assert "AIRFLOW_HOME" in os.environ - - airflow_home = os.environ["AIRFLOW_HOME"] - airflow_db = f"{airflow_home}/airflow.db" - - # reset both resets and initializes a new database - airflow.utils.db.resetdb() - - # Making sure we are using a data file there. - assert os.path.isfile(airflow_db) diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/constraints.txt b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/constraints.txt deleted file mode 100644 index 66b7987b371..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/constraints.txt +++ /dev/null @@ -1,679 +0,0 @@ -# -# This constraints file was automatically generated on 2023-07-07T14:44:02Z -# via "eager-upgrade" mechanism of PIP. For the "v2-6-test" branch of Airflow. -# This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs -# the providers from PIP-released packages at the moment of the constraint generation. -# -# Those constraints are actually those that regular users use to install released version of Airflow. -# We also use those constraints after "apache-airflow" is released and the constraints are tagged with -# "constraints-X.Y.Z" tag to build the production image for that version. -# -# -# This constraints file is meant to be used only in the "apache-airflow" installation command and not -# in all subsequent pip commands. By using a constraints.txt file, we ensure that solely the Airflow -# installation step is reproducible. Subsequent pip commands may install packages that would have -# been incompatible with the constraints used in Airflow reproducible installation step. Finally, pip -# commands that might change the installed version of apache-airflow should include "apache-airflow==X.Y.Z" -# in the list of install targets to prevent Airflow accidental upgrade or downgrade. -# -# Typical installation process of airflow for Python 3.8 is (with random selection of extras and custom -# dependencies added), usually consists of two steps: -# -# 1. Reproducible installation of airflow with selected providers (note constraints are used): -# -# pip install "apache-airflow[celery,cncf.kubernetes,google,amazon,snowflake]==X.Y.Z" \ -# --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-X.Y.Z/constraints-3.8.txt" -# -# 2. Installing own dependencies that are potentially not matching the constraints (note constraints are not -# used, and apache-airflow==X.Y.Z is used to make sure there is no accidental airflow upgrade/downgrade. -# -# pip install "apache-airflow==X.Y.Z" "snowflake-connector-python[pandas]==2.9.0" -# -Authlib==1.2.1 -Babel==2.12.1 -ConfigUpdater==3.1.1 -Deprecated==1.2.14 -Flask-AppBuilder==4.3.1 -Flask-Babel==2.0.0 -Flask-Bcrypt==1.0.1 -Flask-Caching==2.0.2 -Flask-JWT-Extended==4.5.2 -Flask-Limiter==3.3.1 -Flask-Login==0.6.2 -Flask-SQLAlchemy==2.5.1 -Flask-Session==0.5.0 -Flask-WTF==1.1.1 -Flask==2.2.5 -GitPython==3.1.31 -JPype1==1.4.1 -JayDeBeApi==1.2.3 -Jinja2==3.1.2 -Mako==1.2.4 -Markdown==3.4.3 -MarkupSafe==2.1.3 -PyGithub==1.59.0 -PyHive==0.6.5 -PyJWT==2.7.0 -PyNaCl==1.5.0 -PyYAML==6.0 -Pygments==2.15.1 -SQLAlchemy-JSONField==1.0.1.post0 -SQLAlchemy-Utils==0.41.1 -SQLAlchemy==1.4.49 -SecretStorage==3.3.3 -Shapely==1.8.5.post1 -Sphinx==5.3.0 -WTForms==3.0.1 -Werkzeug==2.2.3 -adal==1.2.7 -aiobotocore==2.5.2 -aiofiles==23.1.0 -aiohttp==3.8.4 -aioitertools==0.11.0 -aioresponses==0.7.4 -aiosignal==1.3.1 -alabaster==0.7.13 -alembic==1.11.1 -aliyun-python-sdk-core==2.13.36 -aliyun-python-sdk-kms==2.16.1 -amqp==5.1.1 -analytics-python==1.4.post1 -ansiwrap==0.8.4 -anyascii==0.3.2 -anyio==3.7.1 -apache-airflow-providers-airbyte==3.3.1 -apache-airflow-providers-alibaba==2.4.1 -apache-airflow-providers-amazon==8.2.0 -apache-airflow-providers-apache-beam==5.1.1 -apache-airflow-providers-apache-cassandra==3.2.1 -apache-airflow-providers-apache-drill==2.4.1 -apache-airflow-providers-apache-druid==3.4.1 -apache-airflow-providers-apache-flink==1.1.1 -apache-airflow-providers-apache-hdfs==4.1.0 -apache-airflow-providers-apache-hive==6.1.1 -apache-airflow-providers-apache-impala==1.1.1 -apache-airflow-providers-apache-kylin==3.2.1 -apache-airflow-providers-apache-livy==3.5.1 -apache-airflow-providers-apache-pig==4.1.1 -apache-airflow-providers-apache-pinot==4.1.1 -apache-airflow-providers-apache-spark==4.1.1 -apache-airflow-providers-apache-sqoop==3.2.1 -apache-airflow-providers-arangodb==2.2.1 -apache-airflow-providers-asana==2.2.1 -apache-airflow-providers-atlassian-jira==2.1.1 -apache-airflow-providers-celery==3.2.1 -apache-airflow-providers-cloudant==3.2.1 -apache-airflow-providers-cncf-kubernetes==7.1.0 -apache-airflow-providers-common-sql==1.5.2 -apache-airflow-providers-databricks==4.3.0 -apache-airflow-providers-datadog==3.3.1 -apache-airflow-providers-dbt-cloud==3.2.1 -apache-airflow-providers-dingding==3.2.1 -apache-airflow-providers-discord==3.3.0 -apache-airflow-providers-docker==3.7.1 -apache-airflow-providers-elasticsearch==4.5.1 -apache-airflow-providers-exasol==4.2.1 -apache-airflow-providers-facebook==3.2.1 -apache-airflow-providers-ftp==3.4.2 -apache-airflow-providers-github==2.3.1 -apache-airflow-providers-google==10.2.0 -apache-airflow-providers-grpc==3.2.1 -apache-airflow-providers-hashicorp==3.4.1 -apache-airflow-providers-http==4.4.2 -apache-airflow-providers-imap==3.2.2 -apache-airflow-providers-influxdb==2.2.1 -apache-airflow-providers-jdbc==4.0.0 -apache-airflow-providers-jenkins==3.3.1 -apache-airflow-providers-microsoft-azure==6.1.2 -apache-airflow-providers-microsoft-mssql==3.4.1 -apache-airflow-providers-microsoft-psrp==2.3.1 -apache-airflow-providers-microsoft-winrm==3.2.1 -apache-airflow-providers-mongo==3.2.1 -apache-airflow-providers-mysql==5.1.1 -apache-airflow-providers-neo4j==3.3.1 -apache-airflow-providers-odbc==4.0.0 -apache-airflow-providers-openfaas==3.2.1 -apache-airflow-providers-opsgenie==5.1.1 -apache-airflow-providers-oracle==3.7.1 -apache-airflow-providers-pagerduty==3.3.0 -apache-airflow-providers-papermill==3.2.1 -apache-airflow-providers-plexus==3.2.1 -apache-airflow-providers-postgres==5.5.1 -apache-airflow-providers-presto==5.1.1 -apache-airflow-providers-qubole==3.4.1 -apache-airflow-providers-redis==3.2.1 -apache-airflow-providers-salesforce==5.4.1 -apache-airflow-providers-samba==4.2.1 -apache-airflow-providers-segment==3.2.1 -apache-airflow-providers-sendgrid==3.2.1 -apache-airflow-providers-sftp==4.3.1 -apache-airflow-providers-singularity==3.2.1 -apache-airflow-providers-slack==7.3.1 -apache-airflow-providers-smtp==1.2.0 -apache-airflow-providers-snowflake==4.2.0 -apache-airflow-providers-sqlite==3.4.2 -apache-airflow-providers-ssh==3.7.1 -apache-airflow-providers-tableau==4.2.1 -apache-airflow-providers-tabular==1.2.1 -apache-airflow-providers-telegram==4.1.1 -apache-airflow-providers-trino==5.1.1 -apache-airflow-providers-vertica==3.4.1 -apache-airflow-providers-zendesk==4.3.1 -apache-beam==2.48.0 -apispec==5.2.2 -appdirs==1.4.4 -argcomplete==3.1.1 -arrow==1.2.3 -asana==3.2.1 -asgiref==3.7.2 -asn1crypto==1.5.1 -astroid==2.15.5 -asttokens==2.2.1 -async-timeout==4.0.2 -asynctest==0.13.0 -atlasclient==1.0.0 -atlassian-python-api==3.39.0 -attrs==23.1.0 -aws-sam-translator==1.71.0 -aws-xray-sdk==2.12.0 -azure-batch==14.0.0 -azure-common==1.1.28 -azure-core==1.27.1 -azure-cosmos==4.4.0 -azure-datalake-store==0.0.53 -azure-identity==1.13.0 -azure-keyvault-secrets==4.7.0 -azure-kusto-data==0.0.45 -azure-mgmt-containerinstance==1.5.0 -azure-mgmt-core==1.4.0 -azure-mgmt-datafactory==1.1.0 -azure-mgmt-datalake-nspkg==3.0.1 -azure-mgmt-datalake-store==0.5.0 -azure-mgmt-nspkg==3.0.2 -azure-mgmt-resource==23.0.1 -azure-nspkg==3.0.2 -azure-servicebus==7.11.0 -azure-storage-blob==12.16.0 -azure-storage-common==2.1.0 -azure-storage-file-datalake==12.11.0 -azure-storage-file==2.1.0 -azure-synapse-spark==0.7.0 -backcall==0.2.0 -backoff==1.10.0 -backports.zoneinfo==0.2.1 -bcrypt==4.0.1 -beautifulsoup4==4.12.2 -billiard==4.1.0 -bitarray==2.7.6 -black==23.1a1 -bleach==6.0.0 -blinker==1.6.2 -boto3==1.26.161 -boto==2.49.0 -botocore==1.29.161 -bowler==0.9.0 -cachelib==0.9.0 -cachetools==5.3.1 -cassandra-driver==3.28.0 -cattrs==23.1.2 -celery==5.3.1 -certifi==2023.5.7 -cffi==1.15.1 -cfgv==3.3.1 -cfn-lint==0.77.10 -cgroupspy==0.2.2 -chardet==5.1.0 -charset-normalizer==3.1.0 -checksumdir==1.2.0 -ciso8601==2.3.0 -click-default-group==1.2.2 -click-didyoumean==0.3.0 -click-plugins==1.1.1 -click-repl==0.3.0 -click==8.1.4 -clickclick==20.10.2 -cloudant==2.15.0 -cloudpickle==2.2.1 -colorama==0.4.6 -colorlog==4.8.0 -connexion==2.14.2 -coverage==7.2.7 -crcmod==1.7 -cron-descriptor==1.4.0 -croniter==1.4.1 -cryptography==40.0.2 -curlify==2.2.1 -dask==2023.4.1 -databricks-sql-connector==2.7.0 -datadog==0.45.0 -db-dtypes==1.1.1 -decorator==5.1.1 -defusedxml==0.7.1 -deprecation==2.1.0 -dill==0.3.1.1 -distlib==0.3.6 -distributed==2023.4.1 -dnspython==2.3.0 -docker==6.1.3 -docopt==0.6.2 -docutils==0.20.1 -ecdsa==0.18.0 -elasticsearch-dbapi==0.2.10 -elasticsearch-dsl==7.4.1 -elasticsearch==7.13.4 -email-validator==1.3.1 -entrypoints==0.4 -eralchemy2==1.3.7 -et-xmlfile==1.1.0 -eventlet==0.33.3 -exceptiongroup==1.1.2 -execnet==2.0.0 -executing==1.2.0 -facebook-business==17.0.2 -fastavro==1.8.0 -fasteners==0.18 -fastjsonschema==2.17.1 -filelock==3.12.2 -fissix==21.11.13 -flower==2.0.0 -frozenlist==1.3.3 -fsspec==2023.6.0 -future==0.18.3 -gcloud-aio-auth==4.2.3 -gcloud-aio-bigquery==6.3.0 -gcloud-aio-storage==8.2.0 -gcsfs==2023.6.0 -geomet==0.2.1.post1 -gevent==22.10.2 -gitdb==4.0.10 -google-ads==21.2.0 -google-api-core==2.11.0 -google-api-python-client==2.92.0 -google-auth-httplib2==0.1.0 -google-auth-oauthlib==1.0.0 -google-auth==2.21.0 -google-cloud-aiplatform==1.27.1 -google-cloud-appengine-logging==1.3.1 -google-cloud-audit-log==0.2.5 -google-cloud-automl==2.11.2 -google-cloud-bigquery-datatransfer==3.11.2 -google-cloud-bigquery-storage==2.22.0 -google-cloud-bigquery==3.11.3 -google-cloud-bigtable==2.19.0 -google-cloud-build==3.17.1 -google-cloud-compute==1.12.1 -google-cloud-container==2.26.0 -google-cloud-core==2.3.3 -google-cloud-datacatalog==3.13.1 -google-cloud-dataflow-client==0.8.4 -google-cloud-dataform==0.5.2 -google-cloud-dataplex==1.5.1 -google-cloud-dataproc-metastore==1.12.0 -google-cloud-dataproc==5.4.2 -google-cloud-dlp==3.12.2 -google-cloud-kms==2.18.0 -google-cloud-language==2.10.1 -google-cloud-logging==3.5.0 -google-cloud-memcache==1.7.2 -google-cloud-monitoring==2.15.1 -google-cloud-orchestration-airflow==1.9.1 -google-cloud-os-login==2.9.1 -google-cloud-pubsub==2.17.1 -google-cloud-redis==2.13.1 -google-cloud-resource-manager==1.10.2 -google-cloud-secret-manager==2.16.2 -google-cloud-spanner==3.36.0 -google-cloud-speech==2.21.0 -google-cloud-storage==2.10.0 -google-cloud-tasks==2.13.2 -google-cloud-texttospeech==2.14.1 -google-cloud-translate==3.11.2 -google-cloud-videointelligence==2.11.3 -google-cloud-vision==3.4.4 -google-cloud-workflows==1.10.2 -google-crc32c==1.5.0 -google-re2==1.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.1 -graphql-core==3.2.3 -graphviz==0.20.1 -greenlet==2.0.2 -grpc-google-iam-v1==0.12.6 -grpcio-gcp==0.2.2 -grpcio-status==1.56.0 -grpcio==1.59.3 -gssapi==1.8.2 -gunicorn==20.1.0 -h11==0.14.0 -hdfs==2.7.0 -hmsclient==0.1.1 -httpcore==0.16.3 -httplib2==0.22.0 -httpx==0.23.3 -humanize==4.7.0 -hvac==1.1.1 -identify==2.5.24 -idna==3.4 -ijson==3.2.2 -imagesize==1.4.1 -importlib-metadata==4.13.0 -importlib-resources==5.12.0 -impyla==0.18.0 -incremental==22.10.0 -inflection==0.5.1 -influxdb-client==1.36.1 -iniconfig==2.0.0 -ipdb==0.13.13 -ipython==8.12.2 -isodate==0.6.1 -itsdangerous==2.1.2 -jaraco.classes==3.2.3 -jedi==0.18.2 -jeepney==0.8.0 -jira==3.5.2 -jmespath==0.10.0 -jschema-to-python==1.2.3 -json-merge-patch==0.2 -jsondiff==2.0.0 -jsonpatch==1.33 -jsonpath-ng==1.5.3 -jsonpickle==3.0.1 -jsonpointer==2.4 -jsonschema-spec==0.1.6 -jsonschema-specifications==2023.6.1 -jsonschema==4.18.0 -junit-xml==1.9 -jupyter_client==8.3.0 -jupyter_core==5.3.1 -keyring==24.2.0 -kombu==5.3.1 -krb5==0.5.0 -kubernetes-asyncio==24.2.3 -kubernetes==23.6.0 -kylinpy==2.8.4 -lazy-object-proxy==1.9.0 -ldap3==2.9.1 -limits==3.5.0 -linkify-it-py==2.0.2 -locket==1.0.0 -lockfile==0.12.2 -looker-sdk==23.10.0 -lxml==4.9.3 -lz4==4.3.2 -markdown-it-py==3.0.0 -marshmallow-enum==1.5.1 -marshmallow-oneofschema==3.0.1 -marshmallow-sqlalchemy==0.26.1 -marshmallow==3.19.0 -matplotlib-inline==0.1.6 -mdit-py-plugins==0.4.0 -mdurl==0.1.2 -mongomock==4.1.2 -monotonic==1.6 -more-itertools==9.1.0 -moreorless==0.4.0 -moto==4.1.12 -mpmath==1.3.0 -msal-extensions==1.0.0 -msal==1.22.0 -msgpack==1.0.5 -msrest==0.7.1 -msrestazure==0.6.4 -multi-key-dict==2.0.3 -multidict==6.0.4 -mypy-boto3-appflow==1.28.0 -mypy-boto3-rds==1.28.0 -mypy-boto3-redshift-data==1.28.0 -mypy-boto3-s3==1.28.0 -mypy-extensions==1.0.0 -mypy==1.0.0 -mysqlclient==2.2.0 -nbclient==0.8.0 -nbformat==5.9.0 -neo4j==5.10.0 -networkx==3.1 -nodeenv==1.8.0 -numpy==1.24.4 -oauthlib==3.2.2 -objsize==0.6.1 -openapi-schema-validator==0.4.4 -openapi-spec-validator==0.5.7 -openpyxl==3.1.2 -opentelemetry-api==1.15.0 -opentelemetry-exporter-otlp-proto-grpc==1.15.0 -opentelemetry-exporter-otlp-proto-http==1.15.0 -opentelemetry-exporter-otlp==1.15.0 -opentelemetry-exporter-prometheus==1.12.0rc1 -opentelemetry-proto==1.15.0 -opentelemetry-sdk==1.15.0 -opentelemetry-semantic-conventions==0.36b0 -opsgenie-sdk==2.1.5 -oracledb==1.3.2 -ordered-set==4.1.0 -orjson==3.9.1 -oscrypto==1.3.0 -oss2==2.18.0 -packaging==21.3 -pandas-gbq==0.19.2 -pandas==1.5.3 -papermill==2.4.0 -paramiko==3.2.0 -parso==0.8.3 -partd==1.4.0 -pathable==0.4.3 -pathspec==0.9.0 -pbr==5.11.1 -pdpyras==5.1.0 -pendulum==2.1.2 -pexpect==4.8.0 -pickleshare==0.7.5 -pinotdb==0.5.0 -pipdeptree==2.9.3 -pipx==1.2.0 -pkginfo==1.9.6 -pkgutil_resolve_name==1.3.10 -platformdirs==3.8.1 -pluggy==1.2.0 -ply==3.11 -plyvel==1.5.0 -portalocker==2.7.0 -pre-commit==3.3.3 -presto-python-client==0.8.3 -prison==0.2.1 -prometheus-client==0.17.0 -prompt-toolkit==3.0.39 -proto-plus==1.22.3 -protobuf==4.23.4 -psutil==5.9.5 -psycopg2-binary==2.9.6 -ptyprocess==0.7.0 -pure-eval==0.2.2 -pure-sasl==0.6.2 -py-partiql-parser==0.3.3 -py4j==0.10.9.7 -pyOpenSSL==23.2.0 -pyarrow==11.0.0 -pyasn1-modules==0.2.8 -pyasn1==0.4.8 -pycountry==22.3.5 -pycparser==2.21 -pycryptodome==3.18.0 -pycryptodomex==3.18.0 -pydantic==1.10.11 -pydata-google-auth==1.8.0 -pydot==1.4.2 -pydruid==0.6.5 -pyenchant==3.2.2 -pyexasol==0.25.2 -pygraphviz==1.11 -pyhcl==0.4.4 -pykerberos==1.2.4 -pymongo==4.4.0 -pymssql==2.2.8 -pyodbc==4.0.39 -pyparsing==3.1.0 -pypsrp==0.8.1 -pyrsistent==0.19.3 -pyspark==3.4.1 -pyspnego==0.9.1 -pytest-asyncio==0.21.0 -pytest-capture-warnings==0.0.4 -pytest-cov==4.1.0 -pytest-httpx==0.21.3 -pytest-instafail==0.5.0 -pytest-rerunfailures==12.0 -pytest-timeouts==1.2.1 -pytest-xdist==3.3.1 -pytest==7.4.0 -python-arango==7.5.8 -python-daemon==3.0.1 -python-dateutil==2.8.2 -python-dotenv==1.0.0 -python-http-client==3.3.7 -python-jenkins==1.7.0 -python-jose==3.3.0 -python-ldap==3.4.3 -python-nvd3==0.15.0 -python-slugify==8.0.1 -python-telegram-bot==20.2 -pytz==2023.3 -pytzdata==2020.1 -pywinrm==0.4.3 -pyzmq==25.1.0 -qds-sdk==1.16.1 -reactivex==4.0.4 -readme-renderer==40.0 -redis==4.6.0 -redshift-connector==2.0.912 -referencing==0.29.1 -regex==2023.6.3 -requests-file==1.5.1 -requests-kerberos==0.14.0 -requests-mock==1.11.0 -requests-ntlm==1.2.0 -requests-oauthlib==1.3.1 -requests-toolbelt==1.0.0 -requests==2.31.0 -responses==0.23.1 -rfc3339-validator==0.1.4 -rfc3986==1.5.0 -rich-argparse==1.2.0 -rich-click==1.6.1 -rich==13.4.2 -rpds-py==0.8.8 -rsa==4.9 -ruff==0.0.277 -s3transfer==0.6.1 -sarif-om==1.0.4 -sasl==0.3.1 -scramp==1.4.4 -scrapbook==0.5.0 -semver==3.0.1 -sendgrid==6.10.0 -sentinels==1.0.0 -sentry-sdk==1.27.1 -setproctitle==1.3.2 -simple-salesforce==1.12.4 -six==1.16.0 -slack-sdk==3.21.3 -smbprotocol==1.10.1 -smmap==5.0.0 -sniffio==1.3.0 -snowballstemmer==2.2.0 -snowflake-connector-python==3.0.4 -snowflake-sqlalchemy==1.4.7 -sortedcontainers==2.4.0 -soupsieve==2.4.1 -sphinx-airflow-theme==0.0.12 -sphinx-argparse==0.4.0 -sphinx-autoapi==2.1.1 -sphinx-copybutton==0.5.2 -sphinx-jinja==2.0.2 -sphinx-rtd-theme==1.2.2 -sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.1 -sphinxcontrib-httpdomain==1.8.1 -sphinxcontrib-jquery==4.1 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-redoc==1.6.0 -sphinxcontrib-serializinghtml==1.1.5 -sphinxcontrib-spelling==8.0.0 -spython==0.3.0 -sqlalchemy-bigquery==1.6.1 -sqlalchemy-drill==1.1.2 -sqlalchemy-redshift==0.8.14 -sqlparse==0.4.4 -sshpubkeys==3.3.1 -sshtunnel==0.4.0 -stack-data==0.6.2 -starkbank-ecdsa==2.2.0 -statsd==4.0.1 -sympy==1.12 -tableauserverclient==0.24 -tabulate==0.9.0 -tblib==2.0.0 -tenacity==8.2.2 -termcolor==2.3.0 -text-unidecode==1.3 -textwrap3==0.9.2 -thrift-sasl==0.4.3 -thrift==0.16.0 -time-machine==2.10.0 -tomli==2.0.1 -toolz==0.12.0 -tornado==6.3.2 -towncrier==23.6.0 -tqdm==4.65.0 -traitlets==5.9.0 -trino==0.326.0 -twine==4.0.2 -types-Deprecated==1.2.9.2 -types-Markdown==3.4.2.9 -types-PyMySQL==1.1.0.0 -types-PyYAML==6.0.12.10 -types-boto==2.49.18.8 -types-certifi==2021.10.8.3 -types-croniter==1.4.0.0 -types-docutils==0.20.0.1 -types-paramiko==3.2.0.0 -types-protobuf==4.23.0.1 -types-pyOpenSSL==23.2.0.1 -types-python-dateutil==2.8.19.13 -types-python-slugify==8.0.0.2 -types-pytz==2023.3.0.0 -types-redis==4.6.0.2 -types-requests==2.31.0.1 -types-setuptools==68.0.0.1 -types-tabulate==0.9.0.2 -types-termcolor==1.1.6.2 -types-toml==0.10.8.6 -types-urllib3==1.26.25.13 -typing_extensions==4.7.1 -tzdata==2023.3 -tzlocal==5.0.1 -uc-micro-py==1.0.2 -unicodecsv==0.14.1 -uritemplate==4.1.1 -urllib3==1.26.16 -userpath==1.8.0 -vertica-python==1.3.2 -vine==5.0.0 -virtualenv==20.23.1 -volatile==2.1.0 -watchtower==2.0.1 -wcwidth==0.2.6 -webencodings==0.5.1 -websocket-client==1.6.1 -wrapt==1.15.0 -xmltodict==0.13.0 -yamllint==1.32.0 -yarl==1.9.2 -zeep==4.2.1 -zenpy==2.0.25 -zict==3.0.0 -zipp==3.15.0 -zope.event==5.0 -zope.interface==6.0 -zstandard==0.21.0 \ No newline at end of file diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/__init__.py b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag.py b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag.py deleted file mode 100644 index d3c6a4f5c33..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2021 Google LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# https://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from airflow import models - -from airflow.hooks.base import BaseHook -from airflow.providers.google.cloud.operators.bigquery import BigQueryCheckOperator -from airflow.providers.google.cloud.operators.dataflow import ( - DataflowTemplatedJobStartOperator, -) -from airflow.providers.google.cloud.sensors.gcs import GCSObjectExistenceSensor -from airflow.providers.slack.operators.slack_webhook import SlackWebhookOperator - -from airflow.utils.dates import days_ago -from airflow.utils.state import State - -# Sample data -BUCKET_NAME = "cloud-samples-data/composer/data-orchestration-blog-example" -DATA_FILE_NAME = "bike_station_data.csv" - -# Assumes existence of the following Airflow Variables -PROJECT_ID = "{{var.value.gcp_project}}" -DATASET = "{{var.value.bigquery_dataset}}" -TABLE = "{{var.value.bigquery_table}}" - - -# Slack error notification example taken from Kaxil Naik's blog on Slack Integration: -# https://medium.com/datareply/integrating-slack-alerts-in-airflow-c9dcd155105 -def on_failure_callback(context): - ti = context.get("task_instance") - slack_msg = f""" - :red_circle: Task Failed. - *Task*: {ti.task_id} - *Dag*: {ti.dag_id} - *Execution Time*: {context.get('execution_date')} - *Log Url*: {ti.log_url} - """ - slack_webhook_token = BaseHook.get_connection("slack_connection").password - slack_error = SlackWebhookOperator( - task_id="post_slack_error", - http_conn_id="slack_connection", - channel="#airflow-alerts", - webhook_token=slack_webhook_token, - message=slack_msg, - ) - slack_error.execute(context) - - -with models.DAG( - "dataflow_to_bq_workflow", - schedule_interval=None, - start_date=days_ago(1), - default_args={"on_failure_callback": on_failure_callback}, -) as dag: - validate_file_exists = GCSObjectExistenceSensor( - task_id="validate_file_exists", bucket=BUCKET_NAME, object=DATA_FILE_NAME - ) - - # See Launching Dataflow pipelines with Cloud Composer tutorial for further guidance - # https://cloud.google.com/composer/docs/how-to/using/using-dataflow-template-operator - start_dataflow_job = DataflowTemplatedJobStartOperator( - task_id="start-dataflow-template-job", - job_name="csv_to_bq_transform", - template="gs://dataflow-templates/latest/GCS_Text_to_BigQuery", - parameters={ - "javascriptTextTransformFunctionName": "transform", - "javascriptTextTransformGcsPath": f"gs://{BUCKET_NAME}/udf_transform.js", - "JSONPath": f"gs://{BUCKET_NAME}/bq_schema.json", - "inputFilePattern": f"gs://{BUCKET_NAME}/{DATA_FILE_NAME}", - "bigQueryLoadingTemporaryDirectory": f"gs://{BUCKET_NAME}/tmp/", - "outputTable": f"{PROJECT_ID}:{DATASET}.{TABLE}", - }, - ) - - execute_bigquery_sql = BigQueryCheckOperator( - task_id="execute_bigquery_sql", - sql=f"SELECT COUNT(*) FROM `{PROJECT_ID}.{DATASET}.{TABLE}`", - use_legacy_sql=False, - ) - - validate_file_exists >> start_dataflow_job >> execute_bigquery_sql - - -if __name__ == "__main__": - dag.clear(dag_run_state=State.NONE) - dag.run() diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag_test.py b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag_test.py deleted file mode 100644 index bbe14fdeff7..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag_test.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2021 Google LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# https://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from airflow import models -import internal_unit_testing -import pytest - -# user should substitute their project ID -PROJECT_ID = "your-project-id" -DATASET = "your-bq-output-dataset" -TABLE = "your-bq-output-table" - - -@pytest.fixture(autouse=True, scope="function") -# The fixture `airflow_database` lives in ./conftest.py. -def set_variables(airflow_database): - models.Variable.set("gcp_project", PROJECT_ID) - models.Variable.set("bigquery_dataset", DATASET) - models.Variable.set("bigquery_table", TABLE) - yield - models.Variable.delete("gcp_project") - models.Variable.delete("bigquery_dataset") - models.Variable.delete("bigquery_table") - - -def test_dag_import(): - from . import data_orchestration_blog_sample_dag - - internal_unit_testing.assert_has_valid_dag(data_orchestration_blog_sample_dag) diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/noxfile_config.py b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/noxfile_config.py deleted file mode 100644 index 58ce78689ca..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/noxfile_config.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default TEST_CONFIG_OVERRIDE for python repos. - -# You can copy this file into your directory, then it will be inported from -# the noxfile.py. - -# The source of truth: -# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/noxfile_config.py - -import tempfile - - -# Airflow creates a config file at the installation, so we want to set -# `AIRFLOW_HOME` envvar before running pytest. - -_tmpdir = tempfile.TemporaryDirectory() - - -TEST_CONFIG_OVERRIDE = { - # You can opt out from the test for specific Python versions. - "ignored_versions": [ - "2.7", - "3.6", - "3.7", - "3.9", - "3.10", - "3.11", - ], # Composer w/ Airflow 2 only supports Python 3.8 - # Old samples are opted out of enforcing Python type hints - # All new samples should feature them - "enforce_type_hints": False, - # An envvar key for determining the project id to use. Change it - # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a - # build specific Cloud project. You can also use your own string - # to use your own Cloud project. - "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # If you need to use a specific version of pip, - # change pip_version_override to the string representation - # of the version number, for example, "20.2.4" - "pip_version_override": "20.2.4", - # A dictionary you want to inject into your test. Don't put any - # secrets here. These values will override predefined values. - "envs": {"AIRFLOW_HOME": _tmpdir.name}, -} diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements-test.txt b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements-test.txt deleted file mode 100644 index 4defcdeb160..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements-test.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest==9.1.1 -cloud-composer-dag-test-utils==1.0.0 diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements.txt b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements.txt deleted file mode 100644 index bf1282437f0..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# be sure to update the constraints file to match -# see https://airflow.apache.org/docs/apache-airflow/stable/installation/installing-from-pypi.html#constraints-files -apache-airflow[google]==3.2.1; python_version >= "3.10" -apache-airflow-providers-apache-beam==5.1.1 -apache-airflow-providers-slack==7.3.2 -apache-airflow-providers-http==4.4.2