{
"ab_internal": {
"isEnterprise": false,
"ql": 400,
"requireVersionIncrementsInPullRequests": true,
"sl": 300
},
"allowedHosts": {
"hosts": [
"*.s3.amazonaws.com"
]
},
"connectorBuildOptions": {
"baseImage": "docker.io/airbyte/python-connector-base:4.1.0@sha256:1d1aa21d34e851df4e8a87b391c27724c06e2597608e7161f4d167be853bd7b6"
},
"connectorTestSuitesOptions": [
{
"suite": "unitTests"
},
{
"suite": "liveTests",
"testConnections": [
{
"id": "3dbae341-863a-43c0-bf00-d6a7473c7364",
"name": "s3_100_mb_dev_null"
},
{
"id": "01f508b1-5690-47af-940c-a777d471762f",
"name": "s3_v4_jsonl_config_dev_null"
},
{
"id": "0bc29171-b589-45be-b59c-0bc3138eaf41",
"name": "s3_v4_csv_skip_rows_no_header_config_dev_null"
},
{
"id": "0e53811c-f26c-487c-a493-b068ae663f81",
"name": "s3_v4_parquet_duration_list_struct_config_dev_null"
},
{
"id": "0f06e5a2-da09-4615-a7bc-83a71043866b",
"name": "s3_parquet_dataset_config_dev_null"
},
{
"id": "13d94db7-2e54-44ab-aae8-1b4c96a2f605",
"name": "s3_v4_parquet_decimal_config_dev_null"
},
{
"id": "18e4fed1-f4ca-441f-b5bd-fd80aed40831",
"name": "s3_jsonl_config_dev_null"
},
{
"id": "1a0868c0-1491-43bf-8aa0-9d5ebd36e128",
"name": "s3_v4_avro_duration_config_dev_null"
},
{
"id": "2124602b-0617-4646-bf5f-39482f8fdcee",
"name": "s3_v4_csv_user_schema_cast_complex_config_dev_null"
},
{
"id": "231c0fcb-41e4-414c-980b-4d86f5b8c377",
"name": "s3_v4_csv_user_schema_config_dev_null"
},
{
"id": "277832c6-2554-4cd8-8b5e-85bc19fcfe8a",
"name": "s3_v4_avro_decimal_as_float_config_dev_null"
},
{
"id": "28680b58-f086-4187-9bbe-acef00f11e67",
"name": "s3_v4_csv_custom_encoding_config_dev_null"
},
{
"id": "2d04cf3c-d641-424c-bb88-101a70d8bba8",
"name": "s3_v4_avro_config_dev_null"
},
{
"id": "32976729-6872-4fc5-ad65-1b7024b234b9",
"name": "s3_v4_parquet_config_dev_null"
},
{
"id": "3497d077-2714-42ac-b379-3e02b54ff481",
"name": "s3_jsonl_newlines_config_dev_null"
},
{
"id": "38cb2686-3480-4411-84d3-ff9464342ef3",
"name": "s3_v4_csv_no_header_config_dev_null"
},
{
"id": "40ed8b50-0322-43ef-99d5-1dc4e4995f8a",
"name": "s3_config_dev_null"
},
{
"id": "44884944-3708-489f-b8e2-df7922c9a167",
"name": "s3_unstructured_config_dev_null"
},
{
"id": "47a8291a-56b6-4865-b66f-0f0cba8abdb8",
"name": "s3_v4_csv_with_null_bools_config_dev_null"
},
{
"id": "4dc5f34b-f335-4a93-8816-2064f838d0a0",
"name": "s3_zip_config_avro_dev_null"
},
{
"id": "4f4eb9d4-8b04-4af7-8de0-0e311185fa60",
"name": "s3_v4_parquet_dataset_config_dev_null"
},
{
"id": "5abf0b35-a5b5-40c2-a9c6-f525027aef9e",
"name": "s3_avro_config_dev_null"
},
{
"id": "5b4f179e-a27c-4fbf-bcfe-19b792e8b001",
"name": "s3_v4_parquet_decimal_as_float_config_dev_null"
},
{
"id": "5e0ea12a-0629-4de7-9597-348be8599f32",
"name": "s3_v4_jsonl_newlines_config_dev_null"
},
{
"id": "6e6e9c11-c068-4eae-9faa-f02c643dc52b",
"name": "s3_v4_csv_custom_format_config_dev_null"
},
{
"id": "70e97267-8a4c-48a1-bdef-30609e5d27f9",
"name": "s3_zip_config_csv_custom_encoding_dev_null"
},
{
"id": "8ddea3b4-fb78-44fe-81db-7cdb4e68cae5",
"name": "s3_v4_csv_with_nulls_config_dev_null"
},
{
"id": "8eb9a0f5-1fcc-418a-85f3-8f81c13e0a73",
"name": "s3_v4_avro_decimal_config_dev_null"
},
{
"id": "9e73e8ca-4486-4f14-982c-d26f27792aae",
"name": "s3_zip_config_csv_dev_null"
},
{
"id": "ad846359-1348-4c75-9c7e-1717f42a249e",
"name": "s3_zip_config_jsonl_dev_null"
},
{
"id": "b796ca5b-d5c0-4472-9f63-c277b79f534b",
"name": "s3_zip_config_parquet_dev_null"
},
{
"id": "ba0c3a18-d120-4e29-8fca-467634e00300",
"name": "s3_v4_csv_skip_rows_config_dev_null"
}
]
},
{
"suite": "acceptanceTests",
"testSecrets": [
{
"fileName": "avro_config.json",
"name": "SECRET_SOURCE-S3_AVRO__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_config.json",
"name": "SECRET_SOURCE-S3_CREDS_V4",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "config_iam_role.json",
"name": "SECRET_SOURCE-S3_IAM_ROLE__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "jsonl_newlines_config.json",
"name": "SECRET_SOURCE-S3_JSONL_NEWLINES__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "jsonl_config.json",
"name": "SECRET_SOURCE-S3_JSONL__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "legacy_avro_decimal_config.json",
"name": "SECRET_SOURCE-S3_LEGACY_AVRO_DECIMAL_CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "legacy_avro_duration_config.json",
"name": "SECRET_SOURCE-S3_LEGACY_AVRO_DURATIONS_CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "legacy_csv_custom_encoding_config.json",
"name": "SECRET_SOURCE-S3_LEGACY_CSV_CUSTOM_ENCODING_CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "legacy_csv_custom_format_config.json",
"name": "SECRET_SOURCE-S3_LEGACY_CSV_CUSTOM_FORMAT_CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "legacy_csv_no_header_config.json",
"name": "SECRET_SOURCE-S3_LEGACY_CSV_NO_HEADER_CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "legacy_csv_skip_rows_config.json",
"name": "SECRET_SOURCE-S3_LEGACY_CSV_SKIP_ROWS_CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "legacy_csv_skip_rows_no_header_config.json",
"name": "SECRET_SOURCE-S3_LEGACY_CSV_SKIP_ROWS_NO_HEADER_CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "legacy_csv_user_schema_cast_complex.json",
"name": "SECRET_SOURCE-S3_LEGACY_CSV_USER_SCHEMA_CAST_COMPLEX_CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "legacy_csv_user_schema_config.json",
"name": "SECRET_SOURCE-S3_LEGACY_CSV_USER_SCHEMA_CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "legacy_csv_with_nulls_config.json",
"name": "SECRET_SOURCE-S3_LEGACY_CSV_WITH_NULLS_CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "legacy_csv_with_null_bools_config.json",
"name": "SECRET_SOURCE-S3_LEGACY_CSV_WITH_NULL_BOOLS_CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "legacy_parquet_decimal_config.json",
"name": "SECRET_SOURCE-S3_LEGACY_PARQUET_DECIMAL_CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "legacy_parquet_duration_list_struct_config.json",
"name": "SECRET_SOURCE-S3_LEGACY_PARQUET_DURATION_LIST_STRUCT_CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "parquet_dataset_config.json",
"name": "SECRET_SOURCE-S3_PARQUET_DATASET__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "parquet_config.json",
"name": "SECRET_SOURCE-S3_PARQUET__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "unstructured_config.json",
"name": "SECRET_SOURCE-S3_UNSTRUCTURED__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_avro_decimal_as_float_config.json",
"name": "SECRET_SOURCE-S3_V4_AVRO_DECIMAL_AS_FLOAT__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_avro_decimal_config.json",
"name": "SECRET_SOURCE-S3_V4_AVRO_DECIMAL__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_avro_duration_config.json",
"name": "SECRET_SOURCE-S3_V4_AVRO_DURATIONS__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_avro_config.json",
"name": "SECRET_SOURCE-S3_V4_AVRO__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_csv_custom_encoding_config.json",
"name": "SECRET_SOURCE-S3_V4_CSV_CUSTOM_ENCODING__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_csv_custom_format_config.json",
"name": "SECRET_SOURCE-S3_V4_CSV_CUSTOM_FORMAT__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_csv_no_header_config.json",
"name": "SECRET_SOURCE-S3_V4_CSV_NO_HEADER__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_csv_skip_rows_no_header_config.json",
"name": "SECRET_SOURCE-S3_V4_CSV_SKIP_ROWS_NO_HEADER__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_csv_skip_rows_config.json",
"name": "SECRET_SOURCE-S3_V4_CSV_SKIP_ROWS__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_csv_user_schema_cast_complex_config.json",
"name": "SECRET_SOURCE-S3_V4_CSV_USER_SCHEMA_CAST_COMPLEX__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_csv_user_schema_config.json",
"name": "SECRET_SOURCE-S3_V4_CSV_USER_SCHEMA__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_csv_with_nulls_config.json",
"name": "SECRET_SOURCE-S3_V4_CSV_WITH_NULLS__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_csv_with_null_bools_config.json",
"name": "SECRET_SOURCE-S3_V4_CSV_WITH_NULL_BOOLS__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_jsonl_newlines_config.json",
"name": "SECRET_SOURCE-S3_V4_JSONL_NEWLINE__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_jsonl_config.json",
"name": "SECRET_SOURCE-S3_V4_JSONL__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_parquet_dataset_config.json",
"name": "SECRET_SOURCE-S3_V4_PARQUET_DATASET__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_parquet_decimal_as_float_config.json",
"name": "SECRET_SOURCE-S3_V4_PARQUET_DECIMAL_AS_FLOAT__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_parquet_decimal_config.json",
"name": "SECRET_SOURCE-S3_V4_PARQUET_DECIMAL__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_parquet_duration_list_struct_config.json",
"name": "SECRET_SOURCE-S3_V4_PARQUET_DURATION_LIST_STRUCT__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "v4_parquet_config.json",
"name": "SECRET_SOURCE-S3_V4_PARQUET__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "zip_config_avro.json",
"name": "SECRET_SOURCE-S3_ZIP_AVRO__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "zip_config_csv_custom_encoding.json",
"name": "SECRET_SOURCE-S3_ZIP_CSV_CUSTOM_ENCODING__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "zip_config_csv.json",
"name": "SECRET_SOURCE-S3_ZIP_CSV__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "zip_config_jsonl.json",
"name": "SECRET_SOURCE-S3_ZIP_JSONL__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "zip_config_parquet.json",
"name": "SECRET_SOURCE-S3_ZIP_PARQUET__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
},
{
"fileName": "config.json",
"name": "SECRET_SOURCE-S3__CREDS",
"secretStore": {
"alias": "airbyte-connector-testing-secret-store",
"type": "GSM"
}
}
]
}
],
"custom": false,
"dockerImageTag": "4.15.5-preview.4d61235",
"dockerRepository": "airbyte/source-s3",
"documentationUrl": "https://docs.airbyte.com/integrations/sources/s3",
"externalDocumentationUrls": [
{
"title": "Changelog",
"type": "api_release_history",
"url": "https://docs.aws.amazon.com/AmazonS3/latest/userguide/WhatsNew.html"
}
],
"generated": {
"git": {
"commit_author": "Anatolii Yatsuk",
"commit_author_email": "tolikyatsuk@gmail.com",
"commit_sha": "69bab229e60b983115dc0577bf67b93fd46edb67",
"commit_timestamp": "2026-05-22T22:29:24+03:00"
},
"sbomUrl": "https://connectors.airbyte.com/files/sbom/airbyte/source-s3/4.15.5-preview.4d61235.spdx.json",
"source_file_info": {
"metadata_bucket_name": "prod-airbyte-cloud-connector-metadata-service",
"metadata_file_path": "metadata/airbyte/source-s3/4.15.5-preview.4d61235/metadata.yaml",
"metadata_last_modified": "2026-05-23T05:55:17.693628+00:00",
"registry_entry_generated_at": "2026-05-23T05:55:17.693628+00:00"
}
},
"githubIssueLabel": "source-s3",
"icon": "s3.svg",
"iconUrl": "https://connectors.airbyte.com/files/metadata/airbyte/source-s3/latest/icon.svg",
"language": "python",
"license": "ELv2",
"maxSecondsBetweenMessages": 1,
"name": "S3",
"packageInfo": {
"cdk_version": "python:7.19.2.post3.dev26244645194"
},
"public": true,
"releaseStage": "generally_available",
"releases": {
"breakingChanges": {
"4.0.0": {
"message": "UX improvement, multi-stream support and deprecation of some parsing features",
"migrationDocumentationUrl": "https://docs.airbyte.com/integrations/sources/s3-migrations#4.0.0",
"upgradeDeadline": "2023-10-05"
},
"4.0.4": {
"message": "Following 4.0.0 config change, we are eliminating the `streams.*.file_type` field which was redundant with `streams.*.format`",
"migrationDocumentationUrl": "https://docs.airbyte.com/integrations/sources/s3-migrations#4.0.4",
"upgradeDeadline": "2023-10-18"
}
},
"migrationDocumentationUrl": "https://docs.airbyte.com/integrations/sources/s3-migrations",
"rolloutConfiguration": {
"advanceDelayMinutes": 10,
"enableProgressiveRollout": false,
"initialPercentage": 0,
"maxPercentage": 50
}
},
"remoteRegistries": {
"pypi": {
"enabled": true,
"packageName": "airbyte-source-s3"
}
},
"sourceDefinitionId": "69589781-7828-43c5-9f63-8925b1c1ccc2",
"sourceType": "file",
"spec": {
"connectionSpecification": {
"description": "NOTE: When this Spec is changed, legacy_config_transformer.py must also be modified to uptake the changes\nbecause it is responsible for converting legacy S3 v3 configs into v4 configs using the File-Based CDK.",
"properties": {
"aws_access_key_id": {
"airbyte_secret": true,
"description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.",
"order": 2,
"title": "AWS Access Key ID",
"type": "string"
},
"aws_secret_access_key": {
"airbyte_secret": true,
"description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.",
"order": 3,
"title": "AWS Secret Access Key",
"type": "string"
},
"bucket": {
"description": "Name of the S3 bucket where the file(s) exist.",
"order": 0,
"title": "Bucket",
"type": "string"
},
"dataset": {
"airbyte_hidden": true,
"description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.name instead. The name of the stream you would like this source to output. Can contain letters, numbers, or underscores.",
"order": 100,
"pattern": "^([A-Za-z0-9-_]+)$",
"title": "Output Stream Name",
"type": "string"
},
"delivery_method": {
"default": "use_records_transfer",
"display_type": "radio",
"group": "advanced",
"oneOf": [
{
"description": "Recommended - Extract and load structured records into your destination of choice. This is the classic method of moving data in Airbyte. It allows for blocking and hashing individual fields or files from a structured schema. Data can be flattened, typed and deduped depending on the destination.",
"properties": {
"delivery_type": {
"const": "use_records_transfer",
"default": "use_records_transfer",
"enum": [
"use_records_transfer"
],
"title": "Delivery Type",
"type": "string"
}
},
"required": [
"delivery_type"
],
"title": "Replicate Records",
"type": "object"
},
{
"description": "Copy raw files without parsing their contents. Bits are copied into the destination exactly as they appeared in the source. Recommended for use with unstructured text data, non-text and compressed files.",
"properties": {
"delivery_type": {
"const": "use_file_transfer",
"default": "use_file_transfer",
"enum": [
"use_file_transfer"
],
"title": "Delivery Type",
"type": "string"
},
"preserve_directory_structure": {
"default": true,
"description": "If enabled, sends subdirectory folder structure along with source file names to the destination. Otherwise, files will be synced by their names only. This option is ignored when file-based replication is not enabled.",
"title": "Preserve Sub-Directories in File Paths",
"type": "boolean"
}
},
"required": [
"delivery_type"
],
"title": "Copy Raw Files",
"type": "object"
}
],
"order": 6,
"title": "Delivery Method",
"type": "object"
},
"endpoint": {
"default": "",
"description": "Endpoint to an S3 compatible service. Leave empty to use AWS.",
"examples": [
"my-s3-endpoint.com",
"https://my-s3-endpoint.com"
],
"order": 4,
"title": "Endpoint",
"type": "string"
},
"format": {
"airbyte_hidden": true,
"default": "csv",
"description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.format instead. The format of the files you'd like to replicate",
"oneOf": [
{
"description": "This connector utilises PyArrow (Apache Arrow) for CSV parsing.",
"properties": {
"additional_reader_options": {
"description": "Optionally add a valid JSON string here to provide additional options to the csv reader. Mappings must correspond to options detailed here. 'column_types' is used internally to handle schema so overriding that would likely cause problems.",
"examples": [
"{\"timestamp_parsers\": [\"%m/%d/%Y %H:%M\", \"%Y/%m/%d %H:%M\"], \"strings_can_be_null\": true, \"null_values\": [\"NA\", \"NULL\"]}"
],
"order": 7,
"title": "Additional Reader Options",
"type": "string"
},
"advanced_options": {
"description": "Optionally add a valid JSON string here to provide additional Pyarrow ReadOptions. Specify 'column_names' here if your CSV doesn't have header, or if you want to use custom column names. 'block_size' and 'encoding' are already used above, specify them again here will override the values above.",
"examples": [
"{\"column_names\": [\"column1\", \"column2\"]}"
],
"order": 8,
"title": "Advanced Options",
"type": "string"
},
"block_size": {
"default": 10000,
"description": "The chunk size in bytes to process at a time in memory from each file. If your data is particularly wide and failing during schema detection, increasing this should solve it. Beware of raising this too high as you could hit OOM errors.",
"maximum": 2147483647,
"minimum": 1,
"order": 9,
"title": "Block Size",
"type": "integer"
},
"delimiter": {
"default": ",",
"description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.",
"minLength": 1,
"order": 0,
"title": "Delimiter",
"type": "string"
},
"double_quote": {
"default": true,
"description": "Whether two quotes in a quoted CSV value denote a single quote in the data.",
"order": 5,
"title": "Double Quote",
"type": "boolean"
},
"encoding": {
"default": "utf8",
"description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.",
"order": 4,
"title": "Encoding",
"type": "string"
},
"escape_char": {
"description": "The character used for escaping special characters. To disallow escaping, leave this field blank.",
"order": 3,
"title": "Escape Character",
"type": "string"
},
"filetype": {
"const": "csv",
"default": "csv",
"title": "Filetype",
"type": "string"
},
"infer_datatypes": {
"default": true,
"description": "Configures whether a schema for the source should be inferred from the current data or not. If set to false and a custom schema is set, then the manually enforced schema is used. If a schema is not manually set, and this is set to false, then all fields will be read as strings",
"order": 1,
"title": "Infer Datatypes",
"type": "boolean"
},
"newlines_in_values": {
"default": false,
"description": "Whether newline characters are allowed in CSV values. Turning this on may affect performance. Leave blank to default to False.",
"order": 6,
"title": "Allow newlines in values",
"type": "boolean"
},
"quote_char": {
"default": "\"",
"description": "The character used for quoting CSV values. To disallow quoting, make this field blank.",
"order": 2,
"title": "Quote Character",
"type": "string"
}
},
"title": "CSV",
"type": "object"
},
{
"description": "This connector utilises PyArrow (Apache Arrow) for Parquet parsing.",
"properties": {
"batch_size": {
"default": 65536,
"description": "Maximum number of records per batch read from the input files. Batches may be smaller if there aren\u2019t enough rows in the file. This option can help avoid out-of-memory errors if your data is particularly wide.",
"order": 1,
"title": "Record batch size",
"type": "integer"
},
"buffer_size": {
"default": 2,
"description": "Perform read buffering when deserializing individual column chunks. By default every group column will be loaded fully to memory. This option can help avoid out-of-memory errors if your data is particularly wide.",
"title": "Buffer Size",
"type": "integer"
},
"columns": {
"description": "If you only want to sync a subset of the columns from the file(s), add the columns you want here as a comma-delimited list. Leave it empty to sync all columns.",
"items": {
"type": "string"
},
"order": 0,
"title": "Selected Columns",
"type": "array"
},
"filetype": {
"const": "parquet",
"default": "parquet",
"title": "Filetype",
"type": "string"
}
},
"title": "Parquet",
"type": "object"
},
{
"description": "This connector utilises fastavro for Avro parsing.",
"properties": {
"filetype": {
"const": "avro",
"default": "avro",
"title": "Filetype",
"type": "string"
}
},
"title": "Avro",
"type": "object"
},
{
"description": "This connector uses PyArrow for JSON Lines (jsonl) file parsing.",
"properties": {
"block_size": {
"default": 0,
"description": "The chunk size in bytes to process at a time in memory from each file. If your data is particularly wide and failing during schema detection, increasing this should solve it. Beware of raising this too high as you could hit OOM errors.",
"order": 2,
"title": "Block Size",
"type": "integer"
},
"filetype": {
"const": "jsonl",
"default": "jsonl",
"title": "Filetype",
"type": "string"
},
"newlines_in_values": {
"default": false,
"description": "Whether newline characters are allowed in JSON values. Turning this on may affect performance. Leave blank to default to False.",
"order": 0,
"title": "Allow newlines in values",
"type": "boolean"
},
"unexpected_field_behavior": {
"default": "infer",
"description": "How JSON fields outside of explicit_schema (if given) are treated. Check PyArrow documentation for details",
"enum": [
"ignore",
"infer",
"error"
],
"examples": [
"ignore",
"infer",
"error"
],
"order": 1,
"title": "Unexpected field behavior"
}
},
"title": "Jsonl",
"type": "object"
}
],
"order": 120,
"title": "File Format",
"type": "object"
},
"path_pattern": {
"airbyte_hidden": true,
"description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.globs instead. A regular expression which tells the connector which files to replicate. All files which match this pattern will be replicated. Use | to separate multiple patterns. See this page to understand pattern syntax (GLOBSTAR and SPLIT flags are enabled). Use pattern ** to pick up all files.",
"examples": [
"**",
"myFolder/myTableFiles/*.csv|myFolder/myOtherTableFiles/*.csv"
],
"order": 110,
"title": "Pattern of files to replicate",
"type": "string"
},
"provider": {
"airbyte_hidden": true,
"description": "Deprecated and will be removed soon. Please do not use this field anymore and use bucket, aws_access_key_id, aws_secret_access_key and endpoint instead. Use this to load files from S3 or S3-compatible services",
"order": 111,
"properties": {
"aws_access_key_id": {
"airbyte_secret": true,
"always_show": true,
"description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.",
"order": 1,
"title": "AWS Access Key ID",
"type": "string"
},
"aws_secret_access_key": {
"airbyte_secret": true,
"always_show": true,
"description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.",
"order": 2,
"title": "AWS Secret Access Key",
"type": "string"
},
"bucket": {
"description": "Name of the S3 bucket where the file(s) exist.",
"order": 0,
"title": "Bucket",
"type": "string"
},
"endpoint": {
"default": "",
"description": "Endpoint to an S3 compatible service. Leave empty to use AWS.",
"order": 4,
"title": "Endpoint",
"type": "string"
},
"path_prefix": {
"default": "",
"description": "By providing a path-like prefix (e.g. myFolder/thisTable/) under which all the relevant files sit, we can optimize finding these in S3. This is optional but recommended if your bucket contains many folders/files which you don't need to replicate.",
"order": 3,
"title": "Path Prefix",
"type": "string"
},
"region_name": {
"description": "AWS region where the S3 bucket is located. If not provided, the region will be determined automatically.",
"order": 5,
"title": "AWS Region",
"type": "string"
},
"role_arn": {
"always_show": true,
"description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.",
"order": 7,
"title": "AWS Role ARN",
"type": "string"
},
"start_date": {
"description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any file modified before this date will not be replicated.",
"examples": [
"2021-01-01T00:00:00Z"
],
"format": "date-time",
"order": 6,
"pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
"title": "Start Date",
"type": "string"
}
},
"required": [],
"title": "S3: Amazon Web Services",
"type": "object"
},
"region_name": {
"description": "AWS region where the S3 bucket is located. If not provided, the region will be determined automatically.",
"order": 5,
"title": "AWS Region",
"type": "string"
},
"role_arn": {
"description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.",
"order": 6,
"title": "AWS Role ARN",
"type": "string"
},
"schema": {
"airbyte_hidden": true,
"default": "{}",
"description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.input_schema instead. Optionally provide a schema to enforce, as a valid JSON string. Ensure this is a mapping of { \"column\" : \"type\" }, where types are valid JSON Schema datatypes. Leave as {} to auto-infer the schema.",
"examples": [
"{\"column_1\": \"number\", \"column_2\": \"string\", \"column_3\": \"array\", \"column_4\": \"object\", \"column_5\": \"boolean\"}"
],
"order": 130,
"title": "Manually enforced data schema",
"type": "string"
},
"start_date": {
"description": "UTC date and time in the format 2017-01-25T00:00:00.000000Z. Any file modified before this date will not be replicated.",
"examples": [
"2021-01-01",
"2021-01-01T00:00:00Z",
"2021-01-01T00:00:00.000Z",
"2021-01-01T00:00:00.000000Z"
],
"format": "date-time",
"order": 1,
"pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]+)?(Z|[+-][0-9]{2}:[0-9]{2})?)?$",
"pattern_descriptor": "YYYY-MM-DD, YYYY-MM-DDTHH:mm:ssZ, or YYYY-MM-DDTHH:mm:ss.SSSSSSZ",
"title": "Start Date",
"type": "string"
},
"streams": {
"description": "Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their format, and how they should be parsed and validated. When sending data to warehouse destination such as Snowflake or BigQuery, each stream is a separate table.",
"items": {
"properties": {
"days_to_sync_if_history_is_full": {
"default": 3,
"description": "When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.",
"title": "Days To Sync If History Is Full",
"type": "integer"
},
"format": {
"description": "The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.",
"oneOf": [
{
"properties": {
"double_as_string": {
"default": false,
"description": "Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.",
"title": "Convert Double Fields to Strings",
"type": "boolean"
},
"filetype": {
"const": "avro",
"default": "avro",
"title": "Filetype",
"type": "string"
}
},
"required": [
"filetype"
],
"title": "Avro Format",
"type": "object"
},
{
"properties": {
"delimiter": {
"default": ",",
"description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.",
"title": "Delimiter",
"type": "string"
},
"double_quote": {
"default": true,
"description": "Whether two quotes in a quoted CSV value denote a single quote in the data.",
"title": "Double Quote",
"type": "boolean"
},
"encoding": {
"default": "utf8",
"description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.",
"title": "Encoding",
"type": "string"
},
"escape_char": {
"description": "The character used for escaping special characters. To disallow escaping, leave this field blank.",
"title": "Escape Character",
"type": "string"
},
"false_values": {
"default": [
"n",
"no",
"f",
"false",
"off",
"0"
],
"description": "A set of case-sensitive strings that should be interpreted as false values.",
"items": {
"type": "string"
},
"title": "False Values",
"type": "array",
"uniqueItems": true
},
"filetype": {
"const": "csv",
"default": "csv",
"title": "Filetype",
"type": "string"
},
"header_definition": {
"default": {
"header_definition_type": "From CSV"
},
"description": "How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.",
"oneOf": [
{
"properties": {
"header_definition_type": {
"const": "From CSV",
"default": "From CSV",
"title": "Header Definition Type",
"type": "string"
}
},
"required": [
"header_definition_type"
],
"title": "From CSV",
"type": "object"
},
{
"properties": {
"header_definition_type": {
"const": "Autogenerated",
"default": "Autogenerated",
"title": "Header Definition Type",
"type": "string"
}
},
"required": [
"header_definition_type"
],
"title": "Autogenerated",
"type": "object"
},
{
"properties": {
"column_names": {
"description": "The column names that will be used while emitting the CSV records",
"items": {
"type": "string"
},
"title": "Column Names",
"type": "array"
},
"header_definition_type": {
"const": "User Provided",
"default": "User Provided",
"title": "Header Definition Type",
"type": "string"
}
},
"required": [
"column_names",
"header_definition_type"
],
"title": "User Provided",
"type": "object"
}
],
"title": "CSV Header Definition",
"type": "object"
},
"ignore_errors_on_fields_mismatch": {
"default": false,
"description": "Whether to ignore errors that occur when the number of fields in the CSV does not match the number of columns in the schema.",
"title": "Ignore errors on field mismatch",
"type": "boolean"
},
"inference_type": {
"airbyte_hidden": true,
"default": "None",
"description": "How to infer the types of the columns. If none, inference default to strings.",
"enum": [
"None",
"Primitive Types Only"
],
"title": "Inference Type"
},
"null_values": {
"default": [],
"description": "A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.",
"items": {
"type": "string"
},
"title": "Null Values",
"type": "array",
"uniqueItems": true
},
"quote_char": {
"default": "\"",
"description": "The character used for quoting CSV values. To disallow quoting, make this field blank.",
"title": "Quote Character",
"type": "string"
},
"skip_rows_after_header": {
"default": 0,
"description": "The number of rows to skip after the header row.",
"title": "Skip Rows After Header",
"type": "integer"
},
"skip_rows_before_header": {
"default": 0,
"description": "The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.",
"title": "Skip Rows Before Header",
"type": "integer"
},
"strings_can_be_null": {
"default": true,
"description": "Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.",
"title": "Strings Can Be Null",
"type": "boolean"
},
"true_values": {
"default": [
"y",
"yes",
"t",
"true",
"on",
"1"
],
"description": "A set of case-sensitive strings that should be interpreted as true values.",
"items": {
"type": "string"
},
"title": "True Values",
"type": "array",
"uniqueItems": true
}
},
"required": [
"filetype"
],
"title": "CSV Format",
"type": "object"
},
{
"properties": {
"filetype": {
"const": "jsonl",
"default": "jsonl",
"title": "Filetype",
"type": "string"
}
},
"required": [
"filetype"
],
"title": "Jsonl Format",
"type": "object"
},
{
"properties": {
"decimal_as_float": {
"default": false,
"description": "Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.",
"title": "Convert Decimal Fields to Floats",
"type": "boolean"
},
"filetype": {
"const": "parquet",
"default": "parquet",
"title": "Filetype",
"type": "string"
}
},
"required": [
"filetype"
],
"title": "Parquet Format",
"type": "object"
},
{
"description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.",
"properties": {
"filetype": {
"const": "unstructured",
"default": "unstructured",
"title": "Filetype",
"type": "string"
},
"processing": {
"default": {
"mode": "local"
},
"description": "Processing configuration",
"oneOf": [
{
"description": "Process files locally, supporting `fast` and `ocr` modes. This is the default option.",
"properties": {
"mode": {
"const": "local",
"default": "local",
"enum": [
"local"
],
"title": "Mode",
"type": "string"
}
},
"required": [
"mode"
],
"title": "Local",
"type": "object"
}
],
"title": "Processing",
"type": "object"
},
"skip_unprocessable_files": {
"always_show": true,
"default": true,
"description": "If true, skip files that cannot be parsed and pass the error message along as the _ab_source_file_parse_error field. If false, fail the sync.",
"title": "Skip Unprocessable Files",
"type": "boolean"
},
"strategy": {
"always_show": true,
"default": "auto",
"description": "The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf",
"enum": [
"auto",
"fast",
"ocr_only",
"hi_res"
],
"order": 0,
"title": "Parsing Strategy",
"type": "string"
}
},
"required": [
"filetype"
],
"title": "Unstructured Document Format",
"type": "object"
},
{
"properties": {
"filetype": {
"const": "excel",
"default": "excel",
"title": "Filetype",
"type": "string"
}
},
"required": [
"filetype"
],
"title": "Excel Format",
"type": "object"
}
],
"title": "Format",
"type": "object"
},
"globs": {
"default": [
"**"
],
"description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.",
"items": {
"type": "string"
},
"order": 1,
"title": "Globs",
"type": "array"
},
"input_schema": {
"description": "The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.",
"title": "Input Schema",
"type": "string"
},
"legacy_prefix": {
"airbyte_hidden": true,
"description": "The path prefix configured in v3 versions of the S3 connector. This option is deprecated in favor of a single glob.",
"title": "Legacy Prefix",
"type": "string"
},
"name": {
"description": "The name of the stream.",
"title": "Name",
"type": "string"
},
"primary_key": {
"airbyte_hidden": true,
"description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.",
"title": "Primary Key",
"type": "string"
},
"recent_n_files_to_read_for_schema_discovery": {
"description": "The number of resent files which will be used to discover the schema for this stream.",
"exclusiveMinimum": 0,
"title": "Files To Read For Schema Discover",
"type": "integer"
},
"schemaless": {
"default": false,
"description": "When enabled, syncs will not validate or structure records against the stream's schema.",
"title": "Schemaless",
"type": "boolean"
},
"skip_full_check_for_parquet": {
"default": false,
"description": "When enabled, the CHECK operation for Parquet streams will verify file accessibility but skip the full record-parse step. This avoids out-of-memory errors on large Parquet files.",
"title": "Skip Full Check for Parquet",
"type": "boolean"
},
"use_first_found_file_for_schema_discovery": {
"default": false,
"description": "When enabled, the source will use the first found file for schema discovery. Helps to avoid long discovery step.",
"title": "Use First Found File For Schema Discover",
"type": "boolean"
},
"validation_policy": {
"default": "Emit Record",
"description": "The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.",
"enum": [
"Emit Record",
"Skip Record",
"Wait for Discover"
],
"title": "Validation Policy"
}
},
"required": [
"name",
"format"
],
"title": "FileBasedStreamConfig",
"type": "object"
},
"order": 10,
"title": "The list of streams to sync",
"type": "array"
}
},
"required": [
"streams",
"bucket"
],
"title": "Config",
"type": "object"
},
"documentationUrl": "https://docs.airbyte.com/integrations/sources/s3",
"supportsDBT": false,
"supportsNormalization": false
},
"supportLevel": "certified",
"supportsDataActivation": false,
"supportsFileTransfer": true,
"tags": [
"language:python",
"cdk:python-file-based"
],
"tombstone": false
}