Skip to content

python(feat): Add ParquetUploadService#315

Merged
marc-sift merged 4 commits intomainfrom
parquet-upload-service
Sep 16, 2025
Merged

python(feat): Add ParquetUploadService#315
marc-sift merged 4 commits intomainfrom
parquet-upload-service

Conversation

@marc-sift
Copy link
Contributor

Example:

python main.py 
|████████████████████████████████████████| 24.8k bytes/24.8k bytes [100%] in 0.1s (243.85k bytes/s) 
{
 "data_import_id": "bf5442a2-27f9-4319-88d4-e6b45f861946",
 "created_date": "2025-09-12T21:10:10.448741Z",
 "modified_date": "2025-09-12T21:10:10.493266Z",
 "source_url": "",
 "status": "DATA_IMPORT_STATUS_IN_PROGRESS",
 "error_message": "",
 "csv_config": null,
 "parquet_config": {
  "assetName": "test-parquet",
  "runName": "Example Parquet Upload",
  "runId": "",
  "flatDataset": {
   "timeColumn": {
    "path": "timestamp",
    "format": "TIME_FORMAT_ABSOLUTE_UNIX_NANOSECONDS"
   },
   "dataColumns": [
    {
     "path": "id",
     "channelConfig": {
      "name": "id",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_INT_64",
      "enumTypes": [],
      "bitFieldElements": []
     }
    },
    {
     "path": "struct_channel|field1",
     "channelConfig": {
      "name": "struct_channel.field1",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_INT_64",
      "enumTypes": [],
      "bitFieldElements": []
     }
    },
    {
     "path": "struct_channel|field2",
     "channelConfig": {
      "name": "struct_channel.field2",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_STRING",
      "enumTypes": [],
      "bitFieldElements": []
     }
    },
    {
     "path": "struct_channel|field3|subfield1",
     "channelConfig": {
      "name": "struct_channel.field3.subfield1",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_INT_64",
      "enumTypes": [],
      "bitFieldElements": []
     }
    },
    {
     "path": "struct_channel|field3|subfield2",
     "channelConfig": {
      "name": "struct_channel.field3.subfield2",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_STRING",
      "enumTypes": [],
      "bitFieldElements": []
     }
    },
    {
     "path": "list_channel",
     "channelConfig": {
      "name": "list_channel.json",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_STRING",
      "enumTypes": [],
      "bitFieldElements": []
     }
    },
    {
     "path": "list_channel",
     "channelConfig": {
      "name": "list_channel",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_BYTES",
      "enumTypes": [],
      "bitFieldElements": []
     }
    },
    {
     "path": "map_channel",
     "channelConfig": {
      "name": "map_channel.json",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_STRING",
      "enumTypes": [],
      "bitFieldElements": []
     }
    },
    {
     "path": "map_channel",
     "channelConfig": {
      "name": "map_channel",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_BYTES",
      "enumTypes": [],
      "bitFieldElements": []
     }
    },
    {
     "path": "timestamp",
     "channelConfig": {
      "name": "timestamp",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_INT_64",
      "enumTypes": [],
      "bitFieldElements": []
     }
    },
    {
     "path": "bool_channel",
     "channelConfig": {
      "name": "bool_channel",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_BOOL",
      "enumTypes": [],
      "bitFieldElements": []
     }
    },
    {
     "path": "float_channel",
     "channelConfig": {
      "name": "float_channel",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_DOUBLE",
      "enumTypes": [],
      "bitFieldElements": []
     }
    },
    {
     "path": "string_channel",
     "channelConfig": {
      "name": "string_channel",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_STRING",
      "enumTypes": [],
      "bitFieldElements": []
     }
    },
    {
     "path": "binary_channel",
     "channelConfig": {
      "name": "binary_channel",
      "component": "",
      "units": "",
      "description": "",
      "dataType": "CHANNEL_DATA_TYPE_BYTES",
      "enumTypes": [],
      "bitFieldElements": []
     }
    }
   ]
  },
  "footerOffset": "21593",
  "footerLength": 3228,
  "complexTypesImportMode": "PARQUET_COMPLEX_TYPES_IMPORT_MODE_BOTH"
 }
}
Waiting for upload to complete...
Upload example complete!

@marc-sift marc-sift force-pushed the parquet-upload-service branch 5 times, most recently from 8ed5c13 to 787da5f Compare September 12, 2025 22:17
@marc-sift marc-sift force-pushed the parquet-upload-service branch from 787da5f to c47b00e Compare September 12, 2025 22:27
nathan-sift
nathan-sift previously approved these changes Sep 16, 2025
@marc-sift marc-sift merged commit 8e1c90f into main Sep 16, 2025
10 checks passed
@marc-sift marc-sift deleted the parquet-upload-service branch September 16, 2025 01:04
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants