diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/README.md b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/README.md deleted file mode 100644 index 7d0e250dd81..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Data Orchestration with Cloud Composer - -Code found in this directory is part of a [blog post](https://cloud.google.com/blog/topics/developers-practitioners) published in June 2021 \ No newline at end of file diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/__init__.py b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/conftest.py b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/conftest.py deleted file mode 100644 index e7033247004..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/conftest.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2021 Google LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# https://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import os - -import pytest - - -# this fixture initializes the Airflow DB once per session -# it is used by DAGs in both the blogs and workflows directories, -# unless there exists a conftest at a lower level -@pytest.fixture(scope="session") -def airflow_database(): - import airflow.utils.db - - # We use separate directory for local db path per session - # by setting AIRFLOW_HOME env var, which is done in noxfile_config.py. - - assert "AIRFLOW_HOME" in os.environ - - airflow_home = os.environ["AIRFLOW_HOME"] - airflow_db = f"{airflow_home}/airflow.db" - - # reset both resets and initializes a new database - airflow.utils.db.resetdb() - - # Making sure we are using a data file there. - assert os.path.isfile(airflow_db) diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/constraints.txt b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/constraints.txt deleted file mode 100644 index 66b7987b371..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/constraints.txt +++ /dev/null @@ -1,679 +0,0 @@ -# -# This constraints file was automatically generated on 2023-07-07T14:44:02Z -# via "eager-upgrade" mechanism of PIP. For the "v2-6-test" branch of Airflow. -# This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs -# the providers from PIP-released packages at the moment of the constraint generation. -# -# Those constraints are actually those that regular users use to install released version of Airflow. -# We also use those constraints after "apache-airflow" is released and the constraints are tagged with -# "constraints-X.Y.Z" tag to build the production image for that version. -# -# -# This constraints file is meant to be used only in the "apache-airflow" installation command and not -# in all subsequent pip commands. By using a constraints.txt file, we ensure that solely the Airflow -# installation step is reproducible. Subsequent pip commands may install packages that would have -# been incompatible with the constraints used in Airflow reproducible installation step. Finally, pip -# commands that might change the installed version of apache-airflow should include "apache-airflow==X.Y.Z" -# in the list of install targets to prevent Airflow accidental upgrade or downgrade. -# -# Typical installation process of airflow for Python 3.8 is (with random selection of extras and custom -# dependencies added), usually consists of two steps: -# -# 1. Reproducible installation of airflow with selected providers (note constraints are used): -# -# pip install "apache-airflow[celery,cncf.kubernetes,google,amazon,snowflake]==X.Y.Z" \ -# --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-X.Y.Z/constraints-3.8.txt" -# -# 2. Installing own dependencies that are potentially not matching the constraints (note constraints are not -# used, and apache-airflow==X.Y.Z is used to make sure there is no accidental airflow upgrade/downgrade. -# -# pip install "apache-airflow==X.Y.Z" "snowflake-connector-python[pandas]==2.9.0" -# -Authlib==1.2.1 -Babel==2.12.1 -ConfigUpdater==3.1.1 -Deprecated==1.2.14 -Flask-AppBuilder==4.3.1 -Flask-Babel==2.0.0 -Flask-Bcrypt==1.0.1 -Flask-Caching==2.0.2 -Flask-JWT-Extended==4.5.2 -Flask-Limiter==3.3.1 -Flask-Login==0.6.2 -Flask-SQLAlchemy==2.5.1 -Flask-Session==0.5.0 -Flask-WTF==1.1.1 -Flask==2.2.5 -GitPython==3.1.31 -JPype1==1.4.1 -JayDeBeApi==1.2.3 -Jinja2==3.1.2 -Mako==1.2.4 -Markdown==3.4.3 -MarkupSafe==2.1.3 -PyGithub==1.59.0 -PyHive==0.6.5 -PyJWT==2.7.0 -PyNaCl==1.5.0 -PyYAML==6.0 -Pygments==2.15.1 -SQLAlchemy-JSONField==1.0.1.post0 -SQLAlchemy-Utils==0.41.1 -SQLAlchemy==1.4.49 -SecretStorage==3.3.3 -Shapely==1.8.5.post1 -Sphinx==5.3.0 -WTForms==3.0.1 -Werkzeug==2.2.3 -adal==1.2.7 -aiobotocore==2.5.2 -aiofiles==23.1.0 -aiohttp==3.8.4 -aioitertools==0.11.0 -aioresponses==0.7.4 -aiosignal==1.3.1 -alabaster==0.7.13 -alembic==1.11.1 -aliyun-python-sdk-core==2.13.36 -aliyun-python-sdk-kms==2.16.1 -amqp==5.1.1 -analytics-python==1.4.post1 -ansiwrap==0.8.4 -anyascii==0.3.2 -anyio==3.7.1 -apache-airflow-providers-airbyte==3.3.1 -apache-airflow-providers-alibaba==2.4.1 -apache-airflow-providers-amazon==8.2.0 -apache-airflow-providers-apache-beam==5.1.1 -apache-airflow-providers-apache-cassandra==3.2.1 -apache-airflow-providers-apache-drill==2.4.1 -apache-airflow-providers-apache-druid==3.4.1 -apache-airflow-providers-apache-flink==1.1.1 -apache-airflow-providers-apache-hdfs==4.1.0 -apache-airflow-providers-apache-hive==6.1.1 -apache-airflow-providers-apache-impala==1.1.1 -apache-airflow-providers-apache-kylin==3.2.1 -apache-airflow-providers-apache-livy==3.5.1 -apache-airflow-providers-apache-pig==4.1.1 -apache-airflow-providers-apache-pinot==4.1.1 -apache-airflow-providers-apache-spark==4.1.1 -apache-airflow-providers-apache-sqoop==3.2.1 -apache-airflow-providers-arangodb==2.2.1 -apache-airflow-providers-asana==2.2.1 -apache-airflow-providers-atlassian-jira==2.1.1 -apache-airflow-providers-celery==3.2.1 -apache-airflow-providers-cloudant==3.2.1 -apache-airflow-providers-cncf-kubernetes==7.1.0 -apache-airflow-providers-common-sql==1.5.2 -apache-airflow-providers-databricks==4.3.0 -apache-airflow-providers-datadog==3.3.1 -apache-airflow-providers-dbt-cloud==3.2.1 -apache-airflow-providers-dingding==3.2.1 -apache-airflow-providers-discord==3.3.0 -apache-airflow-providers-docker==3.7.1 -apache-airflow-providers-elasticsearch==4.5.1 -apache-airflow-providers-exasol==4.2.1 -apache-airflow-providers-facebook==3.2.1 -apache-airflow-providers-ftp==3.4.2 -apache-airflow-providers-github==2.3.1 -apache-airflow-providers-google==10.2.0 -apache-airflow-providers-grpc==3.2.1 -apache-airflow-providers-hashicorp==3.4.1 -apache-airflow-providers-http==4.4.2 -apache-airflow-providers-imap==3.2.2 -apache-airflow-providers-influxdb==2.2.1 -apache-airflow-providers-jdbc==4.0.0 -apache-airflow-providers-jenkins==3.3.1 -apache-airflow-providers-microsoft-azure==6.1.2 -apache-airflow-providers-microsoft-mssql==3.4.1 -apache-airflow-providers-microsoft-psrp==2.3.1 -apache-airflow-providers-microsoft-winrm==3.2.1 -apache-airflow-providers-mongo==3.2.1 -apache-airflow-providers-mysql==5.1.1 -apache-airflow-providers-neo4j==3.3.1 -apache-airflow-providers-odbc==4.0.0 -apache-airflow-providers-openfaas==3.2.1 -apache-airflow-providers-opsgenie==5.1.1 -apache-airflow-providers-oracle==3.7.1 -apache-airflow-providers-pagerduty==3.3.0 -apache-airflow-providers-papermill==3.2.1 -apache-airflow-providers-plexus==3.2.1 -apache-airflow-providers-postgres==5.5.1 -apache-airflow-providers-presto==5.1.1 -apache-airflow-providers-qubole==3.4.1 -apache-airflow-providers-redis==3.2.1 -apache-airflow-providers-salesforce==5.4.1 -apache-airflow-providers-samba==4.2.1 -apache-airflow-providers-segment==3.2.1 -apache-airflow-providers-sendgrid==3.2.1 -apache-airflow-providers-sftp==4.3.1 -apache-airflow-providers-singularity==3.2.1 -apache-airflow-providers-slack==7.3.1 -apache-airflow-providers-smtp==1.2.0 -apache-airflow-providers-snowflake==4.2.0 -apache-airflow-providers-sqlite==3.4.2 -apache-airflow-providers-ssh==3.7.1 -apache-airflow-providers-tableau==4.2.1 -apache-airflow-providers-tabular==1.2.1 -apache-airflow-providers-telegram==4.1.1 -apache-airflow-providers-trino==5.1.1 -apache-airflow-providers-vertica==3.4.1 -apache-airflow-providers-zendesk==4.3.1 -apache-beam==2.48.0 -apispec==5.2.2 -appdirs==1.4.4 -argcomplete==3.1.1 -arrow==1.2.3 -asana==3.2.1 -asgiref==3.7.2 -asn1crypto==1.5.1 -astroid==2.15.5 -asttokens==2.2.1 -async-timeout==4.0.2 -asynctest==0.13.0 -atlasclient==1.0.0 -atlassian-python-api==3.39.0 -attrs==23.1.0 -aws-sam-translator==1.71.0 -aws-xray-sdk==2.12.0 -azure-batch==14.0.0 -azure-common==1.1.28 -azure-core==1.27.1 -azure-cosmos==4.4.0 -azure-datalake-store==0.0.53 -azure-identity==1.13.0 -azure-keyvault-secrets==4.7.0 -azure-kusto-data==0.0.45 -azure-mgmt-containerinstance==1.5.0 -azure-mgmt-core==1.4.0 -azure-mgmt-datafactory==1.1.0 -azure-mgmt-datalake-nspkg==3.0.1 -azure-mgmt-datalake-store==0.5.0 -azure-mgmt-nspkg==3.0.2 -azure-mgmt-resource==23.0.1 -azure-nspkg==3.0.2 -azure-servicebus==7.11.0 -azure-storage-blob==12.16.0 -azure-storage-common==2.1.0 -azure-storage-file-datalake==12.11.0 -azure-storage-file==2.1.0 -azure-synapse-spark==0.7.0 -backcall==0.2.0 -backoff==1.10.0 -backports.zoneinfo==0.2.1 -bcrypt==4.0.1 -beautifulsoup4==4.12.2 -billiard==4.1.0 -bitarray==2.7.6 -black==23.1a1 -bleach==6.0.0 -blinker==1.6.2 -boto3==1.26.161 -boto==2.49.0 -botocore==1.29.161 -bowler==0.9.0 -cachelib==0.9.0 -cachetools==5.3.1 -cassandra-driver==3.28.0 -cattrs==23.1.2 -celery==5.3.1 -certifi==2023.5.7 -cffi==1.15.1 -cfgv==3.3.1 -cfn-lint==0.77.10 -cgroupspy==0.2.2 -chardet==5.1.0 -charset-normalizer==3.1.0 -checksumdir==1.2.0 -ciso8601==2.3.0 -click-default-group==1.2.2 -click-didyoumean==0.3.0 -click-plugins==1.1.1 -click-repl==0.3.0 -click==8.1.4 -clickclick==20.10.2 -cloudant==2.15.0 -cloudpickle==2.2.1 -colorama==0.4.6 -colorlog==4.8.0 -connexion==2.14.2 -coverage==7.2.7 -crcmod==1.7 -cron-descriptor==1.4.0 -croniter==1.4.1 -cryptography==40.0.2 -curlify==2.2.1 -dask==2023.4.1 -databricks-sql-connector==2.7.0 -datadog==0.45.0 -db-dtypes==1.1.1 -decorator==5.1.1 -defusedxml==0.7.1 -deprecation==2.1.0 -dill==0.3.1.1 -distlib==0.3.6 -distributed==2023.4.1 -dnspython==2.3.0 -docker==6.1.3 -docopt==0.6.2 -docutils==0.20.1 -ecdsa==0.18.0 -elasticsearch-dbapi==0.2.10 -elasticsearch-dsl==7.4.1 -elasticsearch==7.13.4 -email-validator==1.3.1 -entrypoints==0.4 -eralchemy2==1.3.7 -et-xmlfile==1.1.0 -eventlet==0.33.3 -exceptiongroup==1.1.2 -execnet==2.0.0 -executing==1.2.0 -facebook-business==17.0.2 -fastavro==1.8.0 -fasteners==0.18 -fastjsonschema==2.17.1 -filelock==3.12.2 -fissix==21.11.13 -flower==2.0.0 -frozenlist==1.3.3 -fsspec==2023.6.0 -future==0.18.3 -gcloud-aio-auth==4.2.3 -gcloud-aio-bigquery==6.3.0 -gcloud-aio-storage==8.2.0 -gcsfs==2023.6.0 -geomet==0.2.1.post1 -gevent==22.10.2 -gitdb==4.0.10 -google-ads==21.2.0 -google-api-core==2.11.0 -google-api-python-client==2.92.0 -google-auth-httplib2==0.1.0 -google-auth-oauthlib==1.0.0 -google-auth==2.21.0 -google-cloud-aiplatform==1.27.1 -google-cloud-appengine-logging==1.3.1 -google-cloud-audit-log==0.2.5 -google-cloud-automl==2.11.2 -google-cloud-bigquery-datatransfer==3.11.2 -google-cloud-bigquery-storage==2.22.0 -google-cloud-bigquery==3.11.3 -google-cloud-bigtable==2.19.0 -google-cloud-build==3.17.1 -google-cloud-compute==1.12.1 -google-cloud-container==2.26.0 -google-cloud-core==2.3.3 -google-cloud-datacatalog==3.13.1 -google-cloud-dataflow-client==0.8.4 -google-cloud-dataform==0.5.2 -google-cloud-dataplex==1.5.1 -google-cloud-dataproc-metastore==1.12.0 -google-cloud-dataproc==5.4.2 -google-cloud-dlp==3.12.2 -google-cloud-kms==2.18.0 -google-cloud-language==2.10.1 -google-cloud-logging==3.5.0 -google-cloud-memcache==1.7.2 -google-cloud-monitoring==2.15.1 -google-cloud-orchestration-airflow==1.9.1 -google-cloud-os-login==2.9.1 -google-cloud-pubsub==2.17.1 -google-cloud-redis==2.13.1 -google-cloud-resource-manager==1.10.2 -google-cloud-secret-manager==2.16.2 -google-cloud-spanner==3.36.0 -google-cloud-speech==2.21.0 -google-cloud-storage==2.10.0 -google-cloud-tasks==2.13.2 -google-cloud-texttospeech==2.14.1 -google-cloud-translate==3.11.2 -google-cloud-videointelligence==2.11.3 -google-cloud-vision==3.4.4 -google-cloud-workflows==1.10.2 -google-crc32c==1.5.0 -google-re2==1.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.59.1 -graphql-core==3.2.3 -graphviz==0.20.1 -greenlet==2.0.2 -grpc-google-iam-v1==0.12.6 -grpcio-gcp==0.2.2 -grpcio-status==1.56.0 -grpcio==1.59.3 -gssapi==1.8.2 -gunicorn==20.1.0 -h11==0.14.0 -hdfs==2.7.0 -hmsclient==0.1.1 -httpcore==0.16.3 -httplib2==0.22.0 -httpx==0.23.3 -humanize==4.7.0 -hvac==1.1.1 -identify==2.5.24 -idna==3.4 -ijson==3.2.2 -imagesize==1.4.1 -importlib-metadata==4.13.0 -importlib-resources==5.12.0 -impyla==0.18.0 -incremental==22.10.0 -inflection==0.5.1 -influxdb-client==1.36.1 -iniconfig==2.0.0 -ipdb==0.13.13 -ipython==8.12.2 -isodate==0.6.1 -itsdangerous==2.1.2 -jaraco.classes==3.2.3 -jedi==0.18.2 -jeepney==0.8.0 -jira==3.5.2 -jmespath==0.10.0 -jschema-to-python==1.2.3 -json-merge-patch==0.2 -jsondiff==2.0.0 -jsonpatch==1.33 -jsonpath-ng==1.5.3 -jsonpickle==3.0.1 -jsonpointer==2.4 -jsonschema-spec==0.1.6 -jsonschema-specifications==2023.6.1 -jsonschema==4.18.0 -junit-xml==1.9 -jupyter_client==8.3.0 -jupyter_core==5.3.1 -keyring==24.2.0 -kombu==5.3.1 -krb5==0.5.0 -kubernetes-asyncio==24.2.3 -kubernetes==23.6.0 -kylinpy==2.8.4 -lazy-object-proxy==1.9.0 -ldap3==2.9.1 -limits==3.5.0 -linkify-it-py==2.0.2 -locket==1.0.0 -lockfile==0.12.2 -looker-sdk==23.10.0 -lxml==4.9.3 -lz4==4.3.2 -markdown-it-py==3.0.0 -marshmallow-enum==1.5.1 -marshmallow-oneofschema==3.0.1 -marshmallow-sqlalchemy==0.26.1 -marshmallow==3.19.0 -matplotlib-inline==0.1.6 -mdit-py-plugins==0.4.0 -mdurl==0.1.2 -mongomock==4.1.2 -monotonic==1.6 -more-itertools==9.1.0 -moreorless==0.4.0 -moto==4.1.12 -mpmath==1.3.0 -msal-extensions==1.0.0 -msal==1.22.0 -msgpack==1.0.5 -msrest==0.7.1 -msrestazure==0.6.4 -multi-key-dict==2.0.3 -multidict==6.0.4 -mypy-boto3-appflow==1.28.0 -mypy-boto3-rds==1.28.0 -mypy-boto3-redshift-data==1.28.0 -mypy-boto3-s3==1.28.0 -mypy-extensions==1.0.0 -mypy==1.0.0 -mysqlclient==2.2.0 -nbclient==0.8.0 -nbformat==5.9.0 -neo4j==5.10.0 -networkx==3.1 -nodeenv==1.8.0 -numpy==1.24.4 -oauthlib==3.2.2 -objsize==0.6.1 -openapi-schema-validator==0.4.4 -openapi-spec-validator==0.5.7 -openpyxl==3.1.2 -opentelemetry-api==1.15.0 -opentelemetry-exporter-otlp-proto-grpc==1.15.0 -opentelemetry-exporter-otlp-proto-http==1.15.0 -opentelemetry-exporter-otlp==1.15.0 -opentelemetry-exporter-prometheus==1.12.0rc1 -opentelemetry-proto==1.15.0 -opentelemetry-sdk==1.15.0 -opentelemetry-semantic-conventions==0.36b0 -opsgenie-sdk==2.1.5 -oracledb==1.3.2 -ordered-set==4.1.0 -orjson==3.9.1 -oscrypto==1.3.0 -oss2==2.18.0 -packaging==21.3 -pandas-gbq==0.19.2 -pandas==1.5.3 -papermill==2.4.0 -paramiko==3.2.0 -parso==0.8.3 -partd==1.4.0 -pathable==0.4.3 -pathspec==0.9.0 -pbr==5.11.1 -pdpyras==5.1.0 -pendulum==2.1.2 -pexpect==4.8.0 -pickleshare==0.7.5 -pinotdb==0.5.0 -pipdeptree==2.9.3 -pipx==1.2.0 -pkginfo==1.9.6 -pkgutil_resolve_name==1.3.10 -platformdirs==3.8.1 -pluggy==1.2.0 -ply==3.11 -plyvel==1.5.0 -portalocker==2.7.0 -pre-commit==3.3.3 -presto-python-client==0.8.3 -prison==0.2.1 -prometheus-client==0.17.0 -prompt-toolkit==3.0.39 -proto-plus==1.22.3 -protobuf==4.23.4 -psutil==5.9.5 -psycopg2-binary==2.9.6 -ptyprocess==0.7.0 -pure-eval==0.2.2 -pure-sasl==0.6.2 -py-partiql-parser==0.3.3 -py4j==0.10.9.7 -pyOpenSSL==23.2.0 -pyarrow==11.0.0 -pyasn1-modules==0.2.8 -pyasn1==0.4.8 -pycountry==22.3.5 -pycparser==2.21 -pycryptodome==3.18.0 -pycryptodomex==3.18.0 -pydantic==1.10.11 -pydata-google-auth==1.8.0 -pydot==1.4.2 -pydruid==0.6.5 -pyenchant==3.2.2 -pyexasol==0.25.2 -pygraphviz==1.11 -pyhcl==0.4.4 -pykerberos==1.2.4 -pymongo==4.4.0 -pymssql==2.2.8 -pyodbc==4.0.39 -pyparsing==3.1.0 -pypsrp==0.8.1 -pyrsistent==0.19.3 -pyspark==3.4.1 -pyspnego==0.9.1 -pytest-asyncio==0.21.0 -pytest-capture-warnings==0.0.4 -pytest-cov==4.1.0 -pytest-httpx==0.21.3 -pytest-instafail==0.5.0 -pytest-rerunfailures==12.0 -pytest-timeouts==1.2.1 -pytest-xdist==3.3.1 -pytest==7.4.0 -python-arango==7.5.8 -python-daemon==3.0.1 -python-dateutil==2.8.2 -python-dotenv==1.0.0 -python-http-client==3.3.7 -python-jenkins==1.7.0 -python-jose==3.3.0 -python-ldap==3.4.3 -python-nvd3==0.15.0 -python-slugify==8.0.1 -python-telegram-bot==20.2 -pytz==2023.3 -pytzdata==2020.1 -pywinrm==0.4.3 -pyzmq==25.1.0 -qds-sdk==1.16.1 -reactivex==4.0.4 -readme-renderer==40.0 -redis==4.6.0 -redshift-connector==2.0.912 -referencing==0.29.1 -regex==2023.6.3 -requests-file==1.5.1 -requests-kerberos==0.14.0 -requests-mock==1.11.0 -requests-ntlm==1.2.0 -requests-oauthlib==1.3.1 -requests-toolbelt==1.0.0 -requests==2.31.0 -responses==0.23.1 -rfc3339-validator==0.1.4 -rfc3986==1.5.0 -rich-argparse==1.2.0 -rich-click==1.6.1 -rich==13.4.2 -rpds-py==0.8.8 -rsa==4.9 -ruff==0.0.277 -s3transfer==0.6.1 -sarif-om==1.0.4 -sasl==0.3.1 -scramp==1.4.4 -scrapbook==0.5.0 -semver==3.0.1 -sendgrid==6.10.0 -sentinels==1.0.0 -sentry-sdk==1.27.1 -setproctitle==1.3.2 -simple-salesforce==1.12.4 -six==1.16.0 -slack-sdk==3.21.3 -smbprotocol==1.10.1 -smmap==5.0.0 -sniffio==1.3.0 -snowballstemmer==2.2.0 -snowflake-connector-python==3.0.4 -snowflake-sqlalchemy==1.4.7 -sortedcontainers==2.4.0 -soupsieve==2.4.1 -sphinx-airflow-theme==0.0.12 -sphinx-argparse==0.4.0 -sphinx-autoapi==2.1.1 -sphinx-copybutton==0.5.2 -sphinx-jinja==2.0.2 -sphinx-rtd-theme==1.2.2 -sphinxcontrib-applehelp==1.0.4 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==2.0.1 -sphinxcontrib-httpdomain==1.8.1 -sphinxcontrib-jquery==4.1 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-redoc==1.6.0 -sphinxcontrib-serializinghtml==1.1.5 -sphinxcontrib-spelling==8.0.0 -spython==0.3.0 -sqlalchemy-bigquery==1.6.1 -sqlalchemy-drill==1.1.2 -sqlalchemy-redshift==0.8.14 -sqlparse==0.4.4 -sshpubkeys==3.3.1 -sshtunnel==0.4.0 -stack-data==0.6.2 -starkbank-ecdsa==2.2.0 -statsd==4.0.1 -sympy==1.12 -tableauserverclient==0.24 -tabulate==0.9.0 -tblib==2.0.0 -tenacity==8.2.2 -termcolor==2.3.0 -text-unidecode==1.3 -textwrap3==0.9.2 -thrift-sasl==0.4.3 -thrift==0.16.0 -time-machine==2.10.0 -tomli==2.0.1 -toolz==0.12.0 -tornado==6.3.2 -towncrier==23.6.0 -tqdm==4.65.0 -traitlets==5.9.0 -trino==0.326.0 -twine==4.0.2 -types-Deprecated==1.2.9.2 -types-Markdown==3.4.2.9 -types-PyMySQL==1.1.0.0 -types-PyYAML==6.0.12.10 -types-boto==2.49.18.8 -types-certifi==2021.10.8.3 -types-croniter==1.4.0.0 -types-docutils==0.20.0.1 -types-paramiko==3.2.0.0 -types-protobuf==4.23.0.1 -types-pyOpenSSL==23.2.0.1 -types-python-dateutil==2.8.19.13 -types-python-slugify==8.0.0.2 -types-pytz==2023.3.0.0 -types-redis==4.6.0.2 -types-requests==2.31.0.1 -types-setuptools==68.0.0.1 -types-tabulate==0.9.0.2 -types-termcolor==1.1.6.2 -types-toml==0.10.8.6 -types-urllib3==1.26.25.13 -typing_extensions==4.7.1 -tzdata==2023.3 -tzlocal==5.0.1 -uc-micro-py==1.0.2 -unicodecsv==0.14.1 -uritemplate==4.1.1 -urllib3==1.26.16 -userpath==1.8.0 -vertica-python==1.3.2 -vine==5.0.0 -virtualenv==20.23.1 -volatile==2.1.0 -watchtower==2.0.1 -wcwidth==0.2.6 -webencodings==0.5.1 -websocket-client==1.6.1 -wrapt==1.15.0 -xmltodict==0.13.0 -yamllint==1.32.0 -yarl==1.9.2 -zeep==4.2.1 -zenpy==2.0.25 -zict==3.0.0 -zipp==3.15.0 -zope.event==5.0 -zope.interface==6.0 -zstandard==0.21.0 \ No newline at end of file diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/__init__.py b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag.py b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag.py deleted file mode 100644 index d3c6a4f5c33..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2021 Google LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# https://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from airflow import models - -from airflow.hooks.base import BaseHook -from airflow.providers.google.cloud.operators.bigquery import BigQueryCheckOperator -from airflow.providers.google.cloud.operators.dataflow import ( - DataflowTemplatedJobStartOperator, -) -from airflow.providers.google.cloud.sensors.gcs import GCSObjectExistenceSensor -from airflow.providers.slack.operators.slack_webhook import SlackWebhookOperator - -from airflow.utils.dates import days_ago -from airflow.utils.state import State - -# Sample data -BUCKET_NAME = "cloud-samples-data/composer/data-orchestration-blog-example" -DATA_FILE_NAME = "bike_station_data.csv" - -# Assumes existence of the following Airflow Variables -PROJECT_ID = "{{var.value.gcp_project}}" -DATASET = "{{var.value.bigquery_dataset}}" -TABLE = "{{var.value.bigquery_table}}" - - -# Slack error notification example taken from Kaxil Naik's blog on Slack Integration: -# https://medium.com/datareply/integrating-slack-alerts-in-airflow-c9dcd155105 -def on_failure_callback(context): - ti = context.get("task_instance") - slack_msg = f""" - :red_circle: Task Failed. - *Task*: {ti.task_id} - *Dag*: {ti.dag_id} - *Execution Time*: {context.get('execution_date')} - *Log Url*: {ti.log_url} - """ - slack_webhook_token = BaseHook.get_connection("slack_connection").password - slack_error = SlackWebhookOperator( - task_id="post_slack_error", - http_conn_id="slack_connection", - channel="#airflow-alerts", - webhook_token=slack_webhook_token, - message=slack_msg, - ) - slack_error.execute(context) - - -with models.DAG( - "dataflow_to_bq_workflow", - schedule_interval=None, - start_date=days_ago(1), - default_args={"on_failure_callback": on_failure_callback}, -) as dag: - validate_file_exists = GCSObjectExistenceSensor( - task_id="validate_file_exists", bucket=BUCKET_NAME, object=DATA_FILE_NAME - ) - - # See Launching Dataflow pipelines with Cloud Composer tutorial for further guidance - # https://cloud.google.com/composer/docs/how-to/using/using-dataflow-template-operator - start_dataflow_job = DataflowTemplatedJobStartOperator( - task_id="start-dataflow-template-job", - job_name="csv_to_bq_transform", - template="gs://dataflow-templates/latest/GCS_Text_to_BigQuery", - parameters={ - "javascriptTextTransformFunctionName": "transform", - "javascriptTextTransformGcsPath": f"gs://{BUCKET_NAME}/udf_transform.js", - "JSONPath": f"gs://{BUCKET_NAME}/bq_schema.json", - "inputFilePattern": f"gs://{BUCKET_NAME}/{DATA_FILE_NAME}", - "bigQueryLoadingTemporaryDirectory": f"gs://{BUCKET_NAME}/tmp/", - "outputTable": f"{PROJECT_ID}:{DATASET}.{TABLE}", - }, - ) - - execute_bigquery_sql = BigQueryCheckOperator( - task_id="execute_bigquery_sql", - sql=f"SELECT COUNT(*) FROM `{PROJECT_ID}.{DATASET}.{TABLE}`", - use_legacy_sql=False, - ) - - validate_file_exists >> start_dataflow_job >> execute_bigquery_sql - - -if __name__ == "__main__": - dag.clear(dag_run_state=State.NONE) - dag.run() diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag_test.py b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag_test.py deleted file mode 100644 index bbe14fdeff7..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/dags/data_orchestration_blog_sample_dag_test.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2021 Google LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# https://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from airflow import models -import internal_unit_testing -import pytest - -# user should substitute their project ID -PROJECT_ID = "your-project-id" -DATASET = "your-bq-output-dataset" -TABLE = "your-bq-output-table" - - -@pytest.fixture(autouse=True, scope="function") -# The fixture `airflow_database` lives in ./conftest.py. -def set_variables(airflow_database): - models.Variable.set("gcp_project", PROJECT_ID) - models.Variable.set("bigquery_dataset", DATASET) - models.Variable.set("bigquery_table", TABLE) - yield - models.Variable.delete("gcp_project") - models.Variable.delete("bigquery_dataset") - models.Variable.delete("bigquery_table") - - -def test_dag_import(): - from . import data_orchestration_blog_sample_dag - - internal_unit_testing.assert_has_valid_dag(data_orchestration_blog_sample_dag) diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/noxfile_config.py b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/noxfile_config.py deleted file mode 100644 index 58ce78689ca..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/noxfile_config.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Default TEST_CONFIG_OVERRIDE for python repos. - -# You can copy this file into your directory, then it will be inported from -# the noxfile.py. - -# The source of truth: -# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/noxfile_config.py - -import tempfile - - -# Airflow creates a config file at the installation, so we want to set -# `AIRFLOW_HOME` envvar before running pytest. - -_tmpdir = tempfile.TemporaryDirectory() - - -TEST_CONFIG_OVERRIDE = { - # You can opt out from the test for specific Python versions. - "ignored_versions": [ - "2.7", - "3.6", - "3.7", - "3.9", - "3.10", - "3.11", - ], # Composer w/ Airflow 2 only supports Python 3.8 - # Old samples are opted out of enforcing Python type hints - # All new samples should feature them - "enforce_type_hints": False, - # An envvar key for determining the project id to use. Change it - # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a - # build specific Cloud project. You can also use your own string - # to use your own Cloud project. - "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - # If you need to use a specific version of pip, - # change pip_version_override to the string representation - # of the version number, for example, "20.2.4" - "pip_version_override": "20.2.4", - # A dictionary you want to inject into your test. Don't put any - # secrets here. These values will override predefined values. - "envs": {"AIRFLOW_HOME": _tmpdir.name}, -} diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements-test.txt b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements-test.txt deleted file mode 100644 index 4defcdeb160..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements-test.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytest==9.1.1 -cloud-composer-dag-test-utils==1.0.0 diff --git a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements.txt b/composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements.txt deleted file mode 100644 index bf1282437f0..00000000000 --- a/composer/blog/gcp-tech-blog/data-orchestration-with-composer/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# be sure to update the constraints file to match -# see https://airflow.apache.org/docs/apache-airflow/stable/installation/installing-from-pypi.html#constraints-files -apache-airflow[google]==3.2.1; python_version >= "3.10" -apache-airflow-providers-apache-beam==5.1.1 -apache-airflow-providers-slack==7.3.2 -apache-airflow-providers-http==4.4.2