data-prep-kit/transforms/language/text_encoder/lance-commit-rayjob.yaml at dev · Raghav-Bell/data-prep-kit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
###
# Usage:
# export $(cat .env | xargs)
# envsubst < ededup-rayjob.yaml | kubectl apply -f -
###
apiVersion: ray.io/v1alpha1
kind: RayJob
metadata:
  name: rayjob-lance-commit # Name of your RayCluster
spec:
  shutdownAfterJobFinishes: true
  ttlSecondsAfterFinished: 10
  rayClusterSpec:
    rayVersion: '2.36.1' # Specify the Ray version
    headGroupSpec:
      # Service type for the head node, can be ClusterIP or NodePort
      serviceType: ClusterIP
      # Ray container image for the head node
      rayStartParams:
        dashboard-host: '0.0.0.0'
      template:
        spec:
          containers:
            - name: ray-head
              env:
                - name: DPK_LOG_LEVEL
                  value: DEBUG
                - name: S3_ACCESS_KEY
                  valueFrom:
                    secretKeyRef:
                      name: s3-secret
                      key: s3-key
                - name: S3_SECRET_KEY
                  valueFrom:
                    secretKeyRef:
                      name: s3-secret
                      key: s3-secret
                - name: S3_ENDPOINT
                  valueFrom:
                    secretKeyRef:
                      name: s3-secret
                      key: s3-endpoint
                - name: AWS_ACCESS_KEY_ID
                  valueFrom:
                    secretKeyRef:
                      name: s3-secret
                      key: s3-key
                - name: AWS_SECRET_ACCESS_KEY
                  valueFrom:
                    secretKeyRef:
                      name: s3-secret
                      key: s3-secret
                - name: AWS_ENDPOINT
                  valueFrom:
                    secretKeyRef:
                      name: s3-secret
                      key: s3-endpoint
                - name: HF_HOME
                  value: /tmp/.cache
                - name: WORKER_CPU_REQUEST
                  valueFrom:
                    resourceFieldRef:
                      resource: requests.cpu
              image: quay.io/dataprep1/data-prep-kit/text_encoder-ray:latest
              imagePullPolicy: Always
              ports:
                - containerPort: 6379 # Ray port
                  name: ray-client
                - containerPort: 8265 # Ray Dashboard port
                  name: dashboard
              resources:
                requests:
                  cpu: 4
                  memory: 10Gi
                  ephemeral-storage: 20Gi
                limits:
                  cpu: 4
                  memory: 10Gi
                  ephemeral-storage: 20Gi
  # Job details
  # The command to execute the Python script
  entrypoint: >
    python3 -m dpk_text_encoder.lance_commit
    --lanceDB_storage_type '$LANCEDB_STORAGE_TYPE'
    --lanceDB_uri "$LANCEDB_URI"
    --lanceDB_data_uri "$LANCEDB_DATA_URI"
    --lanceDB_table_name "$LANCEDB_TABLE_NAME"
    --lanceDB_fragments_json_folder "$LANCEDB_FRAGMENTS_JSON_FOLDER"
    --lanceDB_table_schema_folder "$LANCEDB_TABLE_SCHEMA_FOLDER"